# Installing the libraries with the specified version.
!pip install pandas==1.5.3 numpy==1.25.2 matplotlib==3.7.1 seaborn==0.13.1 scikit-learn==1.2.2 statsmodels==0.14.1 -q --user

# Import libraries for data manipulation
import numpy as np
import pandas as pd

# Import libraries for data visualization
import matplotlib.pyplot as plt
import seaborn as sns
import plotly.express as px

sns.set()

# Suppress warnings to prevent them from being displayed during code execution
import warnings
warnings.filterwarnings('ignore')

# Enable the inline plotting of matplotlib figures directly within the notebook
%matplotlib inline

# Removes the limit for the number of displayed columns
pd.set_option("display.max_columns", None)
# Sets the limit for the number of displayed rows
pd.set_option("display.max_rows", 200)
# setting the precision of floating numbers to 5 decimal points
pd.set_option("display.float_format", lambda x: "%.5f" % x)

# Library to split data
from sklearn.model_selection import train_test_split

# To build model for prediction
import statsmodels.stats.api as sms
from statsmodels.stats.outliers_influence import variance_inflation_factor
import statsmodels.api as sm
from statsmodels.tools.tools import add_constant
from sklearn.tree import DecisionTreeClassifier
from sklearn import tree

# To tune different models
from sklearn.model_selection import GridSearchCV


# To get diferent metric scores
from sklearn.metrics import (
    f1_score,
    accuracy_score,
    recall_score,
    precision_score,
    confusion_matrix,
    roc_auc_score,
    precision_recall_curve,
    roc_curve,
    make_scorer,
)

# Suppress warnings to prevent them from being displayed during code execution
import warnings
warnings.filterwarnings('ignore')
from statsmodels.tools.sm_exceptions import ConvergenceWarning
warnings.simplefilter("ignore", ConvergenceWarning)

# Load the dataset
data = pd.read_csv("INNHotelsGroup.csv")

# Copying data to another variable to avoid any changes to original data
df = data.copy()

# Display the first few rows of the dataset
display(df.head())

# Display the last few rows of the dataset
display(df.tail())

# Display the shape of the dataset
df.shape
print("There are", df.shape[0], "rows and", df.shape[1], "columns.\n\n")

# Display the data types of the columns in the dataset
display(df.info())

There are 36275 rows and 19 columns.


<class 'pandas.core.frame.DataFrame'>
RangeIndex: 36275 entries, 0 to 36274
Data columns (total 19 columns):
 #   Column                                Non-Null Count  Dtype  
---  ------                                --------------  -----  
 0   Booking_ID                            36275 non-null  object 
 1   no_of_adults                          36275 non-null  int64  
 2   no_of_children                        36275 non-null  int64  
 3   no_of_weekend_nights                  36275 non-null  int64  
 4   no_of_week_nights                     36275 non-null  int64  
 5   type_of_meal_plan                     36275 non-null  object 
 6   required_car_parking_space            36275 non-null  int64  
 7   room_type_reserved                    36275 non-null  object 
 8   lead_time                             36275 non-null  int64  
 9   arrival_year                          36275 non-null  int64  
 10  arrival_month                         36275 non-null  int64  
 11  arrival_date                          36275 non-null  int64  
 12  market_segment_type                   36275 non-null  object 
 13  repeated_guest                        36275 non-null  int64  
 14  no_of_previous_cancellations          36275 non-null  int64  
 15  no_of_previous_bookings_not_canceled  36275 non-null  int64  
 16  avg_price_per_room                    36275 non-null  float64
 17  no_of_special_requests                36275 non-null  int64  
 18  booking_status                        36275 non-null  object 
dtypes: float64(1), int64(13), object(5)
memory usage: 5.3+ MB

None

# Dropping Booking ID column
df.drop("Booking_ID", axis=1, inplace=True)

# Display the statistical summary for the numerical dataset
df.describe().T

# Display the statistical summary for the categorical dataset
df.describe(include="object").T

# Making a list of all categorical variables ()'object' or 'category')
cat_cols = df.select_dtypes(include=["object", "category"]).columns

# Iterate through each categorical column and print the count of unique categorical levels, followed by a separator line.
for column in cat_cols:
    print(df[column].value_counts())
    print("-" * 50)

type_of_meal_plan
Meal Plan 1     27835
Not Selected     5130
Meal Plan 2      3305
Meal Plan 3         5
Name: count, dtype: int64
--------------------------------------------------
room_type_reserved
Room_Type 1    28130
Room_Type 4     6057
Room_Type 6      966
Room_Type 2      692
Room_Type 5      265
Room_Type 7      158
Room_Type 3        7
Name: count, dtype: int64
--------------------------------------------------
market_segment_type
Online           23214
Offline          10528
Corporate         2017
Complementary      391
Aviation           125
Name: count, dtype: int64
--------------------------------------------------
booking_status
Not_Canceled    24390
Canceled        11885
Name: count, dtype: int64
--------------------------------------------------

# Making a list of all numerical variables ('int64', 'float64', 'complex')
num_cols = df.select_dtypes(include=["int64", "float64", "complex"]).columns

# Iterate through each numerical column and print summary statistics, followed by a separator line.
for column in num_cols:
    print(df[column].describe())
    print("-" * 50)

count   36275.00000
mean        1.84496
std         0.51871
min         0.00000
25%         2.00000
50%         2.00000
75%         2.00000
max         4.00000
Name: no_of_adults, dtype: float64
--------------------------------------------------
count   36275.00000
mean        0.10528
std         0.40265
min         0.00000
25%         0.00000
50%         0.00000
75%         0.00000
max        10.00000
Name: no_of_children, dtype: float64
--------------------------------------------------
count   36275.00000
mean        0.81072
std         0.87064
min         0.00000
25%         0.00000
50%         1.00000
75%         2.00000
max         7.00000
Name: no_of_weekend_nights, dtype: float64
--------------------------------------------------
count   36275.00000
mean        2.20430
std         1.41090
min         0.00000
25%         1.00000
50%         2.00000
75%         3.00000
max        17.00000
Name: no_of_week_nights, dtype: float64
--------------------------------------------------
count   36275.00000
mean        0.03099
std         0.17328
min         0.00000
25%         0.00000
50%         0.00000
75%         0.00000
max         1.00000
Name: required_car_parking_space, dtype: float64
--------------------------------------------------
count   36275.00000
mean       85.23256
std        85.93082
min         0.00000
25%        17.00000
50%        57.00000
75%       126.00000
max       443.00000
Name: lead_time, dtype: float64
--------------------------------------------------
count   36275.00000
mean     2017.82043
std         0.38384
min      2017.00000
25%      2018.00000
50%      2018.00000
75%      2018.00000
max      2018.00000
Name: arrival_year, dtype: float64
--------------------------------------------------
count   36275.00000
mean        7.42365
std         3.06989
min         1.00000
25%         5.00000
50%         8.00000
75%        10.00000
max        12.00000
Name: arrival_month, dtype: float64
--------------------------------------------------
count   36275.00000
mean       15.59700
std         8.74045
min         1.00000
25%         8.00000
50%        16.00000
75%        23.00000
max        31.00000
Name: arrival_date, dtype: float64
--------------------------------------------------
count   36275.00000
mean        0.02564
std         0.15805
min         0.00000
25%         0.00000
50%         0.00000
75%         0.00000
max         1.00000
Name: repeated_guest, dtype: float64
--------------------------------------------------
count   36275.00000
mean        0.02335
std         0.36833
min         0.00000
25%         0.00000
50%         0.00000
75%         0.00000
max        13.00000
Name: no_of_previous_cancellations, dtype: float64
--------------------------------------------------
count   36275.00000
mean        0.15341
std         1.75417
min         0.00000
25%         0.00000
50%         0.00000
75%         0.00000
max        58.00000
Name: no_of_previous_bookings_not_canceled, dtype: float64
--------------------------------------------------
count   36275.00000
mean      103.42354
std        35.08942
min         0.00000
25%        80.30000
50%        99.45000
75%       120.00000
max       540.00000
Name: avg_price_per_room, dtype: float64
--------------------------------------------------
count   36275.00000
mean        0.61966
std         0.78624
min         0.00000
25%         0.00000
50%         0.00000
75%         1.00000
max         5.00000
Name: no_of_special_requests, dtype: float64
--------------------------------------------------

# Function to create labeled barplots
def labeled_barplot(data, feature, figsize=(20, 6), perc=False, n=None):
    """
    Barplot with percentage at the top
    data: dataframe
    feature: dataframe column
    perc: whether to display percentages instead of count (default is False)
    n: displays the top n category levels (default is None, i.e., display all levels)
    """
    total = len(data[feature])  # length of the column
    count = data[feature].nunique()
    plt.figure(figsize=figsize)  # set plot size

    ax = sns.countplot(
        data=data,
        x=feature,
        palette="dark",
        order=data[feature].value_counts().index[:n],
    )

    plt.title(f"Distribution of {feature}", fontsize=16)  # add title
    plt.xlabel(feature, fontsize=14)  # add x-axis label
    plt.ylabel("Count", fontsize=14)  # add y-axis label

    for p in ax.patches:
        if perc:
            label = "{:.1f}%".format(
                100 * p.get_height() / total
            )  # percentage of each class of the category
        else:
            label = p.get_height()  # count of each level of the category
        x = p.get_x() + p.get_width() / 2  # width of the plot
        y = p.get_height()  # height of the plot
        ax.annotate(
            label,
            (x, y),
            ha="center",
            va="center",
            size=12,
            xytext=(0, 5),
            textcoords="offset points",
        )  # annotate the percentage
    plt.xticks(rotation=90, fontsize=12)  # rotate x-axis labels
    plt.yticks(fontsize=12)  # set y-axis labels size
    plt.show()  # show the plot


# Function to display a histogram and boxplot combined
def histogram_boxplot(data, feature, figsize=(12, 6), kde=False, bins=None):
    """
    Boxplot and histogram combined
    data: dataframe
    feature: dataframe column
    figsize: size of figure (default (15,10))
    kde: whether to show the density curve (default False)
    bins: number of bins for histogram (default None)
    """
    f2, (ax_box2, ax_hist2) = plt.subplots(
        nrows=2,  # Number of rows of the subplot grid= 2
        sharex=True,  # x-axis will be shared among all subplots
        gridspec_kw={"height_ratios": (0.25, 0.75)},
        figsize=figsize,
    )  # creating the 2 subplots

    sns.boxplot(
        data=data, x=feature, ax=ax_box2, showmeans=True, color="violet"
    )  # boxplot will be created and a triangle will indicate the mean value of the column

    (
        sns.histplot(data=data, x=feature, kde=kde, ax=ax_hist2, bins=bins)
        if bins
        else sns.histplot(data=data, x=feature, kde=kde, ax=ax_hist2)
    )  # For histogram

    ax_hist2.axvline(
        data[feature].mean(), color="green", linestyle="--", label="Mean"
    )  # Add mean to the histogram
    ax_hist2.axvline(
        data[feature].median(), color="black", linestyle="-", label="Median"
    )  # Add median to the histogram

    ax_box2.set(xlabel="")  # Remove x-axis label for the boxplot
    ax_hist2.set_xlabel(feature, fontsize=14)  # Add x-axis label for the histogram
    ax_hist2.set_ylabel("Count", fontsize=14)  # Add y-axis label for the histogram
    ax_hist2.legend()  # Add legend to the histogram

    plt.title(f"Distribution of {feature}", fontsize=16)  # Add title to the histogram
    plt.show()  # Show the plot

# Let's create a copy of the data
df_eda = df.copy()

# Count the number of bookings per month
monthly_bookings = df_eda["arrival_month"].value_counts().sort_index()

print("Distribution of 'arrival_month'")
print(df_eda["arrival_month"].describe())
labeled_barplot(df_eda, "arrival_month", perc=True)
print("-" * 100)

Distribution of 'arrival_month'
count   36275.00000
mean        7.42365
std         3.06989
min         1.00000
25%         5.00000
50%         8.00000
75%        10.00000
max        12.00000
Name: arrival_month, dtype: float64

----------------------------------------------------------------------------------------------------

# Count the number of bookings per market segment
market_segment_counts = df_eda["market_segment_type"].value_counts()

print("Distribution of 'market_segment_type'")
print(df_eda["market_segment_type"].describe())
labeled_barplot(df_eda, "market_segment_type", perc=True)
print("Market Segment Type Counts:")
print(market_segment_counts)
print("-" * 100)

Distribution of 'market_segment_type'
count      36275
unique         5
top       Online
freq       23214
Name: market_segment_type, dtype: object

Market Segment Type Counts:
market_segment_type
Online           23214
Offline          10528
Corporate         2017
Complementary      391
Aviation           125
Name: count, dtype: int64
----------------------------------------------------------------------------------------------------

# Analyze the differences in room prices by market segment
print("Distribution of 'avg_price_per_room' by 'market_segment_type'")
display(df_eda.groupby("market_segment_type")["avg_price_per_room"].describe())
histogram_boxplot(df_eda, "avg_price_per_room", kde=True)

plt.figure(figsize=(12, 8))
sns.boxplot(
    x="market_segment_type",
    y="avg_price_per_room",
    data=df_eda,
    palette="Set2",
    showmeans=True,
)
plt.title("Room Prices by Market Segment")
plt.xlabel("Market Segment")
plt.ylabel("Average Price per Room")
plt.xticks(rotation=45)
plt.show()
print("-" * 100)

Distribution of 'avg_price_per_room' by 'market_segment_type'

----------------------------------------------------------------------------------------------------

# Calculate the percentage of bookings canceled
cancellation_rate = df_eda["booking_status"].value_counts(normalize=True) * 100

print("Distribution of 'booking_status'")
print(df_eda["booking_status"].describe())
labeled_barplot(df_eda, "booking_status", perc=True)
print("Percentage of Bookings Canceled:")
print(cancellation_rate)
print("-" * 100)

Distribution of 'booking_status'
count            36275
unique               2
top       Not_Canceled
freq             24390
Name: booking_status, dtype: object

Percentage of Bookings Canceled:
booking_status
Not_Canceled   67.23639
Canceled       32.76361
Name: proportion, dtype: float64
----------------------------------------------------------------------------------------------------

# Filter for repeating guests
repeating_guests = df_eda[df_eda["repeated_guest"] == 1]

# Calculate the cancellation rate among repeating guests
repeating_guest_cancellation_rate = (
    repeating_guests["booking_status"].value_counts(normalize=True) * 100
)

print("Distribution of 'booking_status' for Repeating Guests")
print(repeating_guests["booking_status"].describe())
labeled_barplot(repeating_guests, "booking_status", perc=True)
print("Percentage of Repeating Guests Who Canceled:")
print(repeating_guest_cancellation_rate)
print("-" * 100)

Distribution of 'booking_status' for Repeating Guests
count              930
unique               2
top       Not_Canceled
freq               914
Name: booking_status, dtype: object

Percentage of Repeating Guests Who Canceled:
booking_status
Not_Canceled   98.27957
Canceled        1.72043
Name: proportion, dtype: float64
----------------------------------------------------------------------------------------------------

# Analyze the relationship between special requests and booking cancellation
print("Distribution of 'no_of_special_requests' by 'booking_status'")
display(df_eda.groupby("booking_status")["no_of_special_requests"].describe())
histogram_boxplot(df_eda, "no_of_special_requests", kde=True)

plt.figure(figsize=(10, 6))
sns.boxplot(
    x="booking_status",
    y="no_of_special_requests",
    data=df_eda,
    palette="pastel",
    showmeans=True,
)
plt.title("Special Requests vs Booking Cancellation")
plt.xlabel("Booking Status")
plt.ylabel("Number of Special Requests")
plt.show()

# Additionally, calculate the average number of special requests for each booking status
special_requests_cancellation = df_eda.groupby("booking_status")[
    "no_of_special_requests"
].mean()

print("Average Number of Special Requests by Booking Status:")
print(special_requests_cancellation)
print("-" * 100)

Distribution of 'no_of_special_requests' by 'booking_status'

Average Number of Special Requests by Booking Status:
booking_status
Canceled       0.33462
Not_Canceled   0.75855
Name: no_of_special_requests, dtype: float64
----------------------------------------------------------------------------------------------------

# Let's create a copy of the data
df_preprocessing = df_eda.copy()

# Checking missing values across each column
missing_values = df_preprocessing.isnull().sum()
print("The number of missing values on each column of the data set is:")
missing_values

The number of missing values on each column of the data set is:

no_of_adults                            0
no_of_children                          0
no_of_weekend_nights                    0
no_of_week_nights                       0
type_of_meal_plan                       0
required_car_parking_space              0
room_type_reserved                      0
lead_time                               0
arrival_year                            0
arrival_month                           0
arrival_date                            0
market_segment_type                     0
repeated_guest                          0
no_of_previous_cancellations            0
no_of_previous_bookings_not_canceled    0
avg_price_per_room                      0
no_of_special_requests                  0
booking_status                          0
dtype: int64

# Check for complete duplicate records
duplicate_records = df_preprocessing.duplicated().sum()
print("The number of duplicate values on the data set is:", duplicate_records)

The number of duplicate values on the data set is: 10275

# Identify all duplicate rows, including the first occurrence
all_duplicate_rows = df_preprocessing[df_preprocessing.duplicated(keep=False)]

# Display all duplicate rows
display(all_duplicate_rows)

# Check for duplicates in individual columns
for column in df_preprocessing:
    print(f"Duplicates in {column}: {df_preprocessing[column].duplicated().sum()}")
    print("-" * 50)

Duplicates in no_of_adults: 36270
--------------------------------------------------
Duplicates in no_of_children: 36269
--------------------------------------------------
Duplicates in no_of_weekend_nights: 36267
--------------------------------------------------
Duplicates in no_of_week_nights: 36257
--------------------------------------------------
Duplicates in type_of_meal_plan: 36271
--------------------------------------------------
Duplicates in required_car_parking_space: 36273
--------------------------------------------------
Duplicates in room_type_reserved: 36268
--------------------------------------------------
Duplicates in lead_time: 35923
--------------------------------------------------
Duplicates in arrival_year: 36273
--------------------------------------------------
Duplicates in arrival_month: 36263
--------------------------------------------------
Duplicates in arrival_date: 36244
--------------------------------------------------
Duplicates in market_segment_type: 36270
--------------------------------------------------
Duplicates in repeated_guest: 36273
--------------------------------------------------
Duplicates in no_of_previous_cancellations: 36266
--------------------------------------------------
Duplicates in no_of_previous_bookings_not_canceled: 36216
--------------------------------------------------
Duplicates in avg_price_per_room: 32345
--------------------------------------------------
Duplicates in no_of_special_requests: 36269
--------------------------------------------------
Duplicates in booking_status: 36273
--------------------------------------------------

# Check the unique values in the booking_status column
print(data["booking_status"].unique())

['Not_Canceled' 'Canceled']

# Creating new features for total number of nights stayed (weekend + weekday nights)
df_preprocessing["total_nights"] = (
    df_preprocessing["no_of_weekend_nights"] + df_preprocessing["no_of_week_nights"]
)

# Creating a binary feature for whether a customer made any special requests
df_preprocessing["any_special_requests"] = df_preprocessing[
    "no_of_special_requests"
].apply(lambda x: 1 if x > 0 else 0)

# Display the data types of the columns in the dataset
display(df_preprocessing.info())

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 36275 entries, 0 to 36274
Data columns (total 20 columns):
 #   Column                                Non-Null Count  Dtype  
---  ------                                --------------  -----  
 0   no_of_adults                          36275 non-null  int64  
 1   no_of_children                        36275 non-null  int64  
 2   no_of_weekend_nights                  36275 non-null  int64  
 3   no_of_week_nights                     36275 non-null  int64  
 4   type_of_meal_plan                     36275 non-null  object 
 5   required_car_parking_space            36275 non-null  int64  
 6   room_type_reserved                    36275 non-null  object 
 7   lead_time                             36275 non-null  int64  
 8   arrival_year                          36275 non-null  int64  
 9   arrival_month                         36275 non-null  int64  
 10  arrival_date                          36275 non-null  int64  
 11  market_segment_type                   36275 non-null  object 
 12  repeated_guest                        36275 non-null  int64  
 13  no_of_previous_cancellations          36275 non-null  int64  
 14  no_of_previous_bookings_not_canceled  36275 non-null  int64  
 15  avg_price_per_room                    36275 non-null  float64
 16  no_of_special_requests                36275 non-null  int64  
 17  booking_status                        36275 non-null  object 
 18  total_nights                          36275 non-null  int64  
 19  any_special_requests                  36275 non-null  int64  
dtypes: float64(1), int64(15), object(4)
memory usage: 5.5+ MB

None

# Display the first few rows of the dataset
display(df_preprocessing.head())

# Making a list of all numerical variables ('int64', 'float64', 'complex')
num_cols = df_preprocessing.select_dtypes(
    include=["int64", "float64", "complex"]
).columns

# Calculate the number of rows needed for the subplots
num_plots = len(num_cols)
num_rows = (num_plots // 3) + (num_plots % 3 > 0)

# Create subplots
plt.figure(figsize=(15, num_rows * 5))
for i, variable in enumerate(num_cols):
    plt.subplot(num_rows, 3, i + 1)
    sns.boxplot(data=df_preprocessing, x=variable)
    plt.tight_layout(pad=2)
plt.show()

# Check for outliers in continuous variables
outliers = df_preprocessing[num_cols].describe()
print("\nSummary of the numerical features, including outliers:")
display(outliers)

Summary of the numerical features, including outliers:

# Let's create another copy of the data
dfmodeling = df_preprocessing.copy()

from sklearn.preprocessing import LabelEncoder
import pandas as pd
import numpy as np


def identify_and_prepare_data(
    df, target_column, exclude_columns=None, multicollinearity_threshold=0.95
):
    """
    Automatically identify categorical and derived columns, then prepare the data for modeling.

    Parameters:
    df (pd.DataFrame): The input DataFrame.
    target_column (str): The name of the target variable column.
    exclude_columns (list): List of columns to exclude from processing (e.g., ID columns).
    multicollinearity_threshold (float): Threshold for identifying derived columns with high multicollinearity.

    Returns:
    X (pd.DataFrame): The features DataFrame after dropping unnecessary columns.
    y (pd.Series): The target variable series.
    categorical_columns (list): List of identified categorical columns.
    derived_columns (list): List of identified derived columns.
    """

    if exclude_columns is None:
        exclude_columns = []

    # Identify categorical columns
    categorical_columns = df.select_dtypes(
        include=["object", "category"]
    ).columns.tolist()

    # Remove the target column and excluded columns from categorical columns list
    categorical_columns = [
        col
        for col in categorical_columns
        if col not in [target_column] + exclude_columns
    ]

    # Encode categorical columns
    for col in categorical_columns:
        df[f"{col}_encoded"] = LabelEncoder().fit_transform(df[col])

    # Drop original categorical columns after encoding
    df = df.drop(columns=categorical_columns)

    # Identify derived columns based on multicollinearity
    # Ensure we only consider numeric columns for the correlation matrix
    numeric_df = df.select_dtypes(include=[np.number])
    correlation_matrix = numeric_df.corr().abs()

    upper_triangle = correlation_matrix.where(
        np.triu(np.ones(correlation_matrix.shape), k=1).astype(bool)
    )

    # Identify columns with high correlation to drop
    derived_columns = [
        "total_nights"
        # column
        # for column in upper_triangle.columns
        # if any(upper_triangle[column] > multicollinearity_threshold)
    ]

    # Create the list of columns to drop
    columns_to_drop = [target_column] + derived_columns + exclude_columns

    # Drop unnecessary columns
    X = df.drop(columns=columns_to_drop)
    y = df[target_column].apply(lambda x: 1 if x == "Canceled" else 0)

    return X, y, categorical_columns, derived_columns


# Usage Example
# Assuming df_preprocessing is your DataFrame and "booking_status" is the target variable
X, y, identified_categorical_columns, identified_derived_columns = (
    identify_and_prepare_data(
        df=df_preprocessing,
        target_column="booking_status",
        # exclude_columns=["booking_id"],  # Example of an ID column to exclude
    )
)

# Splitting the data into training and testing sets
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42
)

# Outputs: Checking the updated dataset and identified columns
print("First few rows of the training set:")
display(X_train.head())

print("\nFirst few entries of the target variable in the training set:")
display(y_train.head())

print("\nIdentified Categorical Columns:")
print(identified_categorical_columns)

print("\nIdentified Derived Columns:")
print(identified_derived_columns)

First few rows of the training set:

First few entries of the target variable in the training set:

25629    0
14473    0
23720    1
5843     0
18709    1
Name: booking_status, dtype: int64

Identified Categorical Columns:
['type_of_meal_plan', 'room_type_reserved', 'market_segment_type']

Identified Derived Columns:
['total_nights']

# # Data Transformation: Encoding categorical variables
# dfmodeling["type_of_meal_plan_encoded"] = (
#     dfmodeling["type_of_meal_plan"].astype("category").cat.codes
# )
# dfmodeling["room_type_reserved_encoded"] = (
#     dfmodeling["room_type_reserved"].astype("category").cat.codes
# )
# dfmodeling["market_segment_type_encoded"] = (
#     dfmodeling["market_segment_type"].astype("category").cat.codes
# )

# # Separating features and target variable
# X = dfmodeling.drop(
#     columns=[
#         "booking_status",
#         "type_of_meal_plan",
#         "room_type_reserved",
#         "market_segment_type",
#         "total_nights",
#     ]
# )
# y = dfmodeling["booking_status"].apply(
#     lambda x: 1 if x == "Canceled" else 0
# )  # Encoding target variable

# # Splitting the dfmodeling into training and testing sets
# X_train, X_test, y_train, y_test = train_test_split(
#     X, y, test_size=0.2, random_state=42
# )

# # Outputs: Checking the updated dfmodelingset, outliers, and splits
# print("First few rows of the training dfmodeling:")
# display(X_train.head())

# print("\nFirst few entries of the target variable in the training set:")
# display(y_train.head())

# Display the shapes of the training and testing sets
print("Shape of Training set : ", X_train.shape)
print("Shape of test set : ", X_test.shape)

print("\nPercentage of classes in training set:")
display(y_train.value_counts(normalize=True))

print("\nPercentage of classes in test set:")
display(y_test.value_counts(normalize=True))

Shape of Training set :  (29020, 18)
Shape of test set :  (7255, 18)

Percentage of classes in training set:

booking_status
0   0.67371
1   0.32629
Name: proportion, dtype: float64

Percentage of classes in test set:

booking_status
0   0.66699
1   0.33301
Name: proportion, dtype: float64

# defining a function to compute different metrics to check performance of a classification model built using statsmodels
def model_performance_classification_statsmodels(
    model, predictors, target, threshold=0.5
):
    """
    Function to compute different metrics to check classification model performance

    model: classifier
    predictors: independent variables
    target: dependent variable
    threshold: threshold for classifying the observation as class 1
    """

    # checking which probabilities are greater than threshold
    pred_temp = model.predict(predictors) > threshold
    # rounding off the above values to get classes
    pred = np.round(pred_temp)

    acc = accuracy_score(target, pred)  # to compute Accuracy
    recall = recall_score(target, pred)  # to compute Recall
    precision = precision_score(target, pred)  # to compute Precision
    f1 = f1_score(target, pred)  # to compute F1-score

    # creating a dataframe of metrics
    df_perf = pd.DataFrame(
        {
            "Accuracy": acc,
            "Recall": recall,
            "Precision": precision,
            "F1": f1,
        },
        index=[0],
    )

    return df_perf

# defining a function to plot the confusion_matrix of a classification model
def confusion_matrix_statsmodels(model, predictors, target, threshold=0.5):
    """
    To plot the confusion_matrix with percentages and print TN, FP, FN, TP

    model: classifier
    predictors: independent variables
    target: dependent variable
    threshold: threshold for classifying the observation as class 1
    """
    y_pred = model.predict(predictors) > threshold
    cm = confusion_matrix(target, y_pred)

    TN, FP, FN, TP = cm.ravel()

    # Display counts for TN, FP, FN, TP
    print(f"True Negatives (TN): {TN}")
    print(f"False Positives (FP): {FP}")
    print(f"False Negatives (FN): {FN}")
    print(f"True Positives (TP): {TP}")

    labels = np.asarray(
        [
            [f"TN: {TN}\n{TN / cm.sum():.2%}", f"FP: {FP}\n{FP / cm.sum():.2%}"],
            [f"FN: {FN}\n{FN / cm.sum():.2%}", f"TP: {TP}\n{TP / cm.sum():.2%}"],
        ]
    ).reshape(2, 2)

    plt.figure(figsize=(6, 4))
    sns.heatmap(cm, annot=labels, fmt="")
    plt.ylabel("True label")
    plt.xlabel("Predicted label")
    plt.show()

# Building the logistic regression model
logit = sm.Logit(y_train, X_train.astype(float))
lg = logit.fit(disp=False)

print(lg.summary())

                           Logit Regression Results                           
==============================================================================
Dep. Variable:         booking_status   No. Observations:                29020
Model:                          Logit   Df Residuals:                    29002
Method:                           MLE   Df Model:                           17
Date:                Sat, 03 Aug 2024   Pseudo R-squ.:                  0.3097
Time:                        01:50:41   Log-Likelihood:                -12651.
converged:                       True   LL-Null:                       -18327.
Covariance Type:            nonrobust   LLR p-value:                     0.000
========================================================================================================
                                           coef    std err          z      P>|z|      [0.025      0.975]
--------------------------------------------------------------------------------------------------------
no_of_adults                             0.0401      0.034      1.169      0.242      -0.027       0.107
no_of_children                          -0.0503      0.043     -1.164      0.244      -0.135       0.034
no_of_weekend_nights                     0.1338      0.018      7.318      0.000       0.098       0.170
no_of_week_nights                        0.0535      0.011      4.700      0.000       0.031       0.076
required_car_parking_space              -1.5486      0.130    -11.926      0.000      -1.803      -1.294
lead_time                                0.0145      0.000     65.432      0.000       0.014       0.015
arrival_year                            -0.0039   8.16e-05    -48.006      0.000      -0.004      -0.004
arrival_month                           -0.0644      0.006    -11.557      0.000      -0.075      -0.053
arrival_date                             0.0023      0.002      1.289      0.197      -0.001       0.006
repeated_guest                          -1.5182      0.510     -2.974      0.003      -2.519      -0.518
no_of_previous_cancellations             0.1487      0.072      2.079      0.038       0.009       0.289
no_of_previous_bookings_not_canceled    -0.0259      0.071     -0.363      0.717      -0.166       0.114
avg_price_per_room                       0.0196      0.001     31.034      0.000       0.018       0.021
no_of_special_requests                  -1.0567      0.055    -19.318      0.000      -1.164      -0.949
any_special_requests                    -0.4749      0.075     -6.358      0.000      -0.621      -0.329
type_of_meal_plan_encoded                0.1087      0.015      7.022      0.000       0.078       0.139
room_type_reserved_encoded              -0.0850      0.014     -5.908      0.000      -0.113      -0.057
market_segment_type_encoded              1.2783      0.039     32.879      0.000       1.202       1.354
========================================================================================================

# Plot the confusion matrix for the training data
confusion_matrix_statsmodels(lg, X_train, y_train)

# Print the performance metrics for the training data
print("Training performance:")
logit_performance_metrics_train = model_performance_classification_statsmodels(
    lg, X_train, y_train
)
logit_performance_metrics_train

True Negatives (TN): 17543
False Positives (FP): 2008
False Negatives (FN): 3669
True Positives (TP): 5800

Training performance:

# Plot the confusion matrix for the testing data
confusion_matrix_statsmodels(lg, X_test, y_test)

# Print the performance metrics for the training data
print("Testing performance:")
logit_performance_metrics_test = model_performance_classification_statsmodels(
    lg, X_test, y_test
)
logit_performance_metrics_test

True Negatives (TN): 4352
False Positives (FP): 487
False Negatives (FN): 936
True Positives (TP): 1480

Testing performance:

# Function to calculate VIF for each feature
def calculate_vif(X):
    vif_data = pd.DataFrame()
    vif_data["feature"] = X.columns
    vif_data["VIF"] = [
        variance_inflation_factor(X.values, i) for i in range(X.shape[1])
    ]
    return vif_data


# Function to iteratively remove features with high VIF
def remove_high_vif_features(X, threshold=5.0):
    while True:
        vif = calculate_vif(X)
        max_vif = vif["VIF"].max()
        if max_vif > threshold:
            feature_to_remove = vif.loc[vif["VIF"].idxmax(), "feature"]
            print(
                f"Removing feature with high VIF: {feature_to_remove} (VIF = {max_vif:.2f})"
            )
            X = X.drop(columns=[feature_to_remove])
        else:
            break
    return X, vif

# Calculate VIF for the features
vif_before = calculate_vif(X_train)
print(vif_before)

                                 feature      VIF
0                           no_of_adults 17.65198
1                         no_of_children  1.34301
2                   no_of_weekend_nights  1.97664
3                      no_of_week_nights  3.76298
4             required_car_parking_space  1.06870
5                              lead_time  2.23127
6                           arrival_year 53.55953
7                          arrival_month  7.23461
8                           arrival_date  4.20731
9                         repeated_guest  1.69811
10          no_of_previous_cancellations  1.35802
11  no_of_previous_bookings_not_canceled  1.61317
12                    avg_price_per_room 15.57239
13                no_of_special_requests  6.53970
14                  any_special_requests  7.34724
15             type_of_meal_plan_encoded  1.41589
16            room_type_reserved_encoded  1.91665
17           market_segment_type_encoded 44.68421

# Iteratively remove features with high VIF
X_train_vif_reduced, final_vif = remove_high_vif_features(X_train)
print("Final VIF values after removing high multicollinearity features:")
print(final_vif)

Removing feature with high VIF: arrival_year (VIF = 53.56)
Removing feature with high VIF: market_segment_type_encoded (VIF = 26.94)
Removing feature with high VIF: no_of_adults (VIF = 13.12)
Removing feature with high VIF: avg_price_per_room (VIF = 8.32)
Removing feature with high VIF: any_special_requests (VIF = 7.09)
Final VIF values after removing high multicollinearity features:
                                 feature     VIF
0                         no_of_children 1.23818
1                   no_of_weekend_nights 1.88363
2                      no_of_week_nights 3.29695
3             required_car_parking_space 1.05835
4                              lead_time 2.12983
5                          arrival_month 4.55520
6                           arrival_date 3.19793
7                         repeated_guest 1.53025
8           no_of_previous_cancellations 1.34571
9   no_of_previous_bookings_not_canceled 1.60304
10                no_of_special_requests 1.72007
11             type_of_meal_plan_encoded 1.27720
12            room_type_reserved_encoded 1.53312

X_trained_multicoll = X_train_vif_reduced

logit2_multicoll = sm.Logit(y_train, X_trained_multicoll.astype(float))
lg2 = logit2_multicoll.fit(disp=False)

# Plot the confusion matrix for the training data
confusion_matrix_statsmodels(lg2, X_trained_multicoll, y_train)

# Print the performance metrics for the new training data
print("Training performance:")
multicoll_performance_metrics_train = model_performance_classification_statsmodels(
    lg2, X_trained_multicoll, y_train
)
display(multicoll_performance_metrics_train)

# Align the testing data with the reduced training data (after removing high VIF features)
X_test_vif_reduced = X_test[X_train_vif_reduced.columns]
X_test_multicoll = X_test_vif_reduced

# Predict on the testing data
y_test_pred = lg2.predict(X_test_multicoll.astype(float))

# Plot the confusion matrix for the testing data
confusion_matrix_statsmodels(lg2, X_test_multicoll, y_test)

# Print the performance metrics for the testing data
print("Testing performance:")
multicoll_performance_metrics_train = model_performance_classification_statsmodels(
    lg2, X_test_multicoll, y_test
)
display(multicoll_performance_metrics_train)

True Negatives (TN): 16938
False Positives (FP): 2613
False Negatives (FN): 4474
True Positives (TP): 4995

Training performance:

True Negatives (TN): 4206
False Positives (FP): 633
False Negatives (FN): 1154
True Positives (TP): 1262

Testing performance:

def remove_high_p_value_variables(X_train, y_train, significance_level=0.05):
    """
    Iteratively remove variables with p-values greater than the significance level from the logistic regression model.
    Displays the initial and final p-values before and after the iteration.

    Parameters:
    X_train (pd.DataFrame): The features DataFrame for the training set.
    y_train (pd.Series): The target variable for the training set.
    significance_level (float): The p-value threshold for keeping variables (default is 0.05).

    Returns:
    final_selected_features (pd.DataFrame): The DataFrame of selected features after removing variables with high p-values.
    final_p_values (pd.Series): The final p-values after the iteration.
    """

    # Initial list of columns
    cols = X_train.columns.tolist()

    # Fit the initial logistic regression model
    initial_model = sm.Logit(y_train, X_train.astype(float)).fit(disp=False)

    # Display initial p-values
    initial_p_values = initial_model.pvalues
    print("Initial p-values:")
    display(initial_p_values)

    while len(cols) > 0:
        # Define the training set with the current list of columns
        X_train_aux = X_train[cols]

        # Fit the logistic regression model
        model = sm.Logit(y_train, X_train_aux.astype(float)).fit(disp=False)

        # Get the p-values and find the maximum p-value
        p_values = model.pvalues
        max_p_value = max(p_values)

        # Name of the variable with the highest p-value
        feature_with_p_max = p_values.idxmax()

        # Check if the highest p-value exceeds the significance level
        if max_p_value > significance_level:
            print(f"Dropping '{feature_with_p_max}' with p-value {max_p_value:.5f}")
            cols.remove(
                feature_with_p_max
            )  # Remove the feature with the highest p-value
        else:
            break  # Stop if all p-values are below the significance level

    # Selected features after the iterative process
    final_selected_features = X_train[cols]

    # Fit the final model with the selected features
    final_model = sm.Logit(y_train, final_selected_features.astype(float)).fit(
        disp=False
    )

    # Display final p-values
    final_p_values = final_model.pvalues
    print("\nFinal p-values after iteration:")
    display(final_p_values)

    return final_selected_features, final_p_values


# Running the function
X_train_pvalues, final_p_values = remove_high_p_value_variables(
    X_trained_multicoll, y_train, significance_level=0.05
)

Initial p-values:

no_of_children                         0.00000
no_of_weekend_nights                   0.00552
no_of_week_nights                      0.00000
required_car_parking_space             0.00000
lead_time                              0.00000
arrival_month                          0.00000
arrival_date                           0.00000
repeated_guest                         0.00000
no_of_previous_cancellations           0.00228
no_of_previous_bookings_not_canceled   0.25613
no_of_special_requests                 0.00000
type_of_meal_plan_encoded              0.00000
room_type_reserved_encoded             0.00000
dtype: float64

Dropping 'no_of_previous_bookings_not_canceled' with p-value 0.25613

Final p-values after iteration:

no_of_children                 0.00000
no_of_weekend_nights           0.00554
no_of_week_nights              0.00000
required_car_parking_space     0.00000
lead_time                      0.00000
arrival_month                  0.00000
arrival_date                   0.00000
repeated_guest                 0.00000
no_of_previous_cancellations   0.00227
no_of_special_requests         0.00000
type_of_meal_plan_encoded      0.00000
room_type_reserved_encoded     0.00000
dtype: float64

logit3_pvalues = sm.Logit(y_train, X_train_pvalues.astype(float))
lg3 = logit3_pvalues.fit(disp=False)

# Plot the confusion matrix for the training data
confusion_matrix_statsmodels(lg3, X_train_pvalues, y_train)

# Print the performance metrics for the new training data
print("Training performance:")
pvalues_performance_metrics_train = model_performance_classification_statsmodels(
    lg3, X_train_pvalues, y_train
)
display(pvalues_performance_metrics_train)

# Align the testing data with the reduced training data features (after removing high p-value variables)
X_test_pvalues = X_test_multicoll[X_train_pvalues.columns]

# Predict on the testing data and evaluate the model performance
# Plot the confusion matrix for the testing data
confusion_matrix_statsmodels(lg3, X_test_pvalues, y_test)

# Print the performance metrics for the testing data
print("Testing performance:")
pvalues_performance_metrics_test = model_performance_classification_statsmodels(
    lg3, X_test_pvalues, y_test
)
display(pvalues_performance_metrics_test)

True Negatives (TN): 16939
False Positives (FP): 2612
False Negatives (FN): 4472
True Positives (TP): 4997

Training performance:

True Negatives (TN): 4206
False Positives (FP): 633
False Negatives (FN): 1154
True Positives (TP): 1262

Testing performance:

X_train_final = X_train_pvalues
X_test_final = X_test_pvalues

# Building the logistic regression model
logit_final = logit3_pvalues
lg_final = logit_final.fit(disp=False)

print(lg_final.summary())

                           Logit Regression Results                           
==============================================================================
Dep. Variable:         booking_status   No. Observations:                29020
Model:                          Logit   Df Residuals:                    29008
Method:                           MLE   Df Model:                           11
Date:                Sat, 03 Aug 2024   Pseudo R-squ.:                  0.2094
Time:                        01:50:44   Log-Likelihood:                -14490.
converged:                       True   LL-Null:                       -18327.
Covariance Type:            nonrobust   LLR p-value:                     0.000
================================================================================================
                                   coef    std err          z      P>|z|      [0.025      0.975]
------------------------------------------------------------------------------------------------
no_of_children                   0.3098      0.038      8.253      0.000       0.236       0.383
no_of_weekend_nights             0.0457      0.016      2.774      0.006       0.013       0.078
no_of_week_nights               -0.0525      0.010     -5.291      0.000      -0.072      -0.033
required_car_parking_space      -1.2321      0.123     -9.983      0.000      -1.474      -0.990
lead_time                        0.0115      0.000     59.742      0.000       0.011       0.012
arrival_month                   -0.1314      0.004    -33.002      0.000      -0.139      -0.124
arrival_date                    -0.0223      0.001    -15.458      0.000      -0.025      -0.019
repeated_guest                  -3.3823      0.408     -8.299      0.000      -4.181      -2.584
no_of_previous_cancellations     0.1992      0.065      3.052      0.002       0.071       0.327
no_of_special_requests          -0.9907      0.023    -42.449      0.000      -1.036      -0.945
type_of_meal_plan_encoded        0.1495      0.014     10.831      0.000       0.122       0.177
room_type_reserved_encoded       0.1678      0.011     14.741      0.000       0.145       0.190
================================================================================================

# converting coefficients to odds
odds = np.exp(lg_final.params)

# finding the percentage change
perc_change_odds = (np.exp(lg_final.params) - 1) * 100

# removing limit from number of columns to display
pd.set_option("display.max_columns", None)

# adding the odds to a dataframe
pd.DataFrame(
    {"Odds": odds, "Change_odd%": perc_change_odds}, index=X_train_final.columns
).T

# Plot the confusion matrix for the training data
confusion_matrix_statsmodels(lg_final, X_train_final, y_train)

# Print the performance metrics for the new training data
print("Training performance:")
log_reg_model_train_perf = model_performance_classification_statsmodels(
    lg_final, X_train_final, y_train
)
display(log_reg_model_train_perf)

True Negatives (TN): 16939
False Positives (FP): 2612
False Negatives (FN): 4472
True Positives (TP): 4997

Training performance:

# Calculate ROC-AUC for the training set
logit_roc_auc_train = roc_auc_score(y_train, lg_final.predict(X_train_final))

# Get the false positive rate and true positive rate
fpr, tpr, thresholds = roc_curve(y_train, lg_final.predict(X_train_final))

# Plot the ROC curve
plt.figure(figsize=(7, 5))
plt.plot(fpr, tpr, label="Logistic Regression (area = %0.2f)" % logit_roc_auc_train)
plt.plot([0, 1], [0, 1], "r--")  # Diagonal line
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.01])
plt.xlabel("False Positive Rate")
plt.ylabel("True Positive Rate")
plt.title("Receiver Operating Characteristic (ROC) Curve")
plt.legend(loc="lower right")
plt.show()

print("Train Set ROC-AUC:", logit_roc_auc_train)

Train Set ROC-AUC: 0.8008376066777733

# Calculate the false positive rate (fpr), true positive rate (tpr), and thresholds
fpr, tpr, thresholds = roc_curve(y_train, lg_final.predict(X_train_final))

# Find the optimal index where the difference between TPR and FPR is maximized
optimal_idx = np.argmax(tpr - fpr)
optimal_threshold_auc_roc = thresholds[optimal_idx]
print(f"Optimal Threshold: {optimal_threshold_auc_roc:.4f}")

Optimal Threshold: 0.3643

# Create the confusion matrix using the optimal threshold
confusion_matrix_statsmodels(
    lg_final, X_train_final, y_train, threshold=optimal_threshold_auc_roc
)

True Negatives (TN): 14555
False Positives (FP): 4996
False Negatives (FN): 2686
True Positives (TP): 6783

# Check the model performance with the new threshold
log_reg_model_train_perf_threshold_auc_roc = (
    model_performance_classification_statsmodels(
        lg_final, X_train_final, y_train, threshold=optimal_threshold_auc_roc
    )
)
print("Training performance with optimal threshold:")
display(log_reg_model_train_perf_threshold_auc_roc)

Training performance with optimal threshold:

# Predict the probabilities on the training data
y_scores = lg_final.predict(X_train_final)

# Generate the precision, recall, and threshold values
prec, rec, tre = precision_recall_curve(y_train, y_scores)


# Function to plot precision and recall against the thresholds
def plot_prec_recall_vs_tresh(precisions, recalls, thresholds):
    plt.plot(thresholds, precisions[:-1], "b--", label="Precision")
    plt.plot(thresholds, recalls[:-1], "g--", label="Recall")
    plt.xlabel("Threshold")
    plt.legend(loc="upper left")
    plt.ylim([0, 1])
    plt.title("Precision and Recall vs Threshold")


# Plot the Precision-Recall curve
plt.figure(figsize=(10, 7))
plot_prec_recall_vs_tresh(prec, rec, tre)
plt.show()

# Calculate the F1 scores for each threshold
f1_scores = 2 * (prec * rec) / (prec + rec)

# Find the threshold that gives the maximum F1 score
optimal_idx = np.argmax(f1_scores)
optimal_threshold_curve = tre[optimal_idx]
print("Optimal Threshold based on Precision-Recall Curve:", optimal_threshold_curve)

Optimal Threshold based on Precision-Recall Curve: 0.3463375314218031

# Use the optimal threshold to create a confusion matrix
confusion_matrix_statsmodels(
    lg_final, X_train_final, y_train, threshold=optimal_threshold_curve
)

# Check the model performance with the new threshold
log_reg_model_train_perf_threshold_curve = model_performance_classification_statsmodels(
    lg_final, X_train_final, y_train, threshold=optimal_threshold_curve
)
print("Training performance with optimal threshold from Precision-Recall curve:")
display(log_reg_model_train_perf_threshold_curve)

True Negatives (TN): 14091
False Positives (FP): 5460
False Negatives (FN): 2465
True Positives (TP): 7004

Training performance with optimal threshold from Precision-Recall curve:

# Predict on the testing data and evaluate the model performance
# Plot the confusion matrix for the testing data
confusion_matrix_statsmodels(lg_final, X_test_final, y_test)

True Negatives (TN): 4206
False Positives (FP): 633
False Negatives (FN): 1154
True Positives (TP): 1262

# Checking the model performance on the test set
log_reg_model_test_perf = model_performance_classification_statsmodels(
    lg_final, X_test_final, y_test
)

# Print the test set performance metrics
print("Test performance:")
display(log_reg_model_test_perf)

Test performance:

# Calculate ROC-AUC for the test set
logit_roc_auc_test = roc_auc_score(y_test, lg_final.predict(X_test_final))

# Get the false positive rate and true positive rate for the test set
fpr_test, tpr_test, thresholds_test = roc_curve(y_test, lg_final.predict(X_test_final))

# Plot the ROC curve for the test set
plt.figure(figsize=(7, 5))
plt.plot(
    fpr_test, tpr_test, label="Logistic Regression (area = %0.2f)" % logit_roc_auc_test
)
plt.plot([0, 1], [0, 1], "r--")  # Diagonal line representing random guessing
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.01])
plt.xlabel("False Positive Rate")
plt.ylabel("True Positive Rate")
plt.title("Receiver Operating Characteristic (ROC) Curve - Test Set")
plt.legend(loc="lower right")
plt.show()

print("Test Set ROC-AUC:", logit_roc_auc_test)

Test Set ROC-AUC: 0.8018190279996004

# Use the optimal threshold found in the training phase
confusion_matrix_statsmodels(
    lg_final, X_test_final, y_test, threshold=optimal_threshold_curve
)

# Check the model performance on the test set with the new threshold
log_reg_model_test_perf_threshold = model_performance_classification_statsmodels(
    lg_final, X_test_final, y_test, threshold=optimal_threshold_curve
)

print("Test performance with optimal threshold from training set:")
display(log_reg_model_test_perf_threshold)

True Negatives (TN): 3509
False Positives (FP): 1330
False Negatives (FN): 624
True Positives (TP): 1792

Test performance with optimal threshold from training set:

# Precision-Recall curve for test set
y_scores_test = lg_final.predict(X_test_final)
prec_test, rec_test, thresholds_test = precision_recall_curve(y_test, y_scores_test)


# Plot Precision-Recall curve
def plot_prec_recall_vs_tresh(precisions, recalls, thresholds):
    plt.plot(thresholds, precisions[:-1], "b--", label="precision")
    plt.plot(thresholds, recalls[:-1], "g--", label="recall")
    plt.xlabel("Threshold")
    plt.legend(loc="upper left")
    plt.ylim([0, 1])


plt.figure(figsize=(10, 7))
plot_prec_recall_vs_tresh(prec_test, rec_test, thresholds_test)
plt.title("Precision-Recall Curve - Test Set")
plt.show()

# Get the predicted probabilities for the test set
y_scores_test = lg_final.predict(X_test_final)

# Calculate precision, recall, and thresholds
prec_test, rec_test, thresholds_test = precision_recall_curve(y_test, y_scores_test)

# Calculate F1 scores for each threshold
f1_scores_test = 2 * (prec_test * rec_test) / (prec_test + rec_test)

# Find the threshold that gives the maximum F1 score
optimal_idx_test = np.argmax(f1_scores_test)
optimal_threshold_test = thresholds_test[optimal_idx_test]
print(
    "Optimal Threshold based on Precision-Recall Curve for Test Set:",
    optimal_threshold_test,
)

Optimal Threshold based on Precision-Recall Curve for Test Set: 0.3576726321099713

# Use the new optimal threshold to create a confusion matrix for the test set
confusion_matrix_statsmodels(
    lg_final, X_test_final, y_test, threshold=optimal_threshold_test
)

# Check the model performance on the test set with the new threshold
log_reg_model_test_perf_new_threshold = model_performance_classification_statsmodels(
    lg_final, X_test_final, y_test, threshold=optimal_threshold_test
)

print(
    "Test performance with optimal threshold from Precision-Recall curve on Test Set:"
)
display(log_reg_model_test_perf_new_threshold)

True Negatives (TN): 3602
False Positives (FP): 1237
False Negatives (FN): 658
True Positives (TP): 1758

Test performance with optimal threshold from Precision-Recall curve on Test Set:

# Define your threshold values
default_threshold_train = 0.5  # The default threshold for logistic regression
optimal_threshold_auc_roc_train = (
    optimal_threshold_auc_roc  # Example threshold from ROC-AUC
)
optimal_threshold_curve_train = (
    optimal_threshold_curve  # Example threshold from Precision-Recall
)

# Create a DataFrame comparing training performance metrics for different thresholds
models_train_comp_df = pd.concat(
    [
        log_reg_model_train_perf.T,  # Performance with the default threshold
        log_reg_model_train_perf_threshold_auc_roc.T,  # Performance with optimal threshold from ROC-AUC
        log_reg_model_train_perf_threshold_curve.T,  # Performance with optimal threshold from Precision-Recall
    ],
    axis=1,
)

# Rename columns to reflect the threshold used
models_train_comp_df.columns = [
    f"Logistic Regression-default Threshold ({default_threshold_train})",
    f"Logistic Regression-Threshold from ROC-AUC ({optimal_threshold_auc_roc_train})",
    f"Logistic Regression-Threshold from Precision-Recall ({optimal_threshold_curve_train})",
]

# Display the comparison
print("Training performance comparison:")
display(models_train_comp_df)

Training performance comparison:

# Define your threshold values
default_threshold_test = 0.5  # The default threshold for logistic regression
optimal_threshold_auc_roc_test = (
    optimal_threshold_curve  # Example threshold from ROC-AUC (if used for test set)
)
optimal_threshold_curve_test = optimal_threshold_test  # Example threshold from Precision-Recall (if used for test set)

# Create a DataFrame comparing test performance metrics for different thresholds
models_test_comp_df = pd.concat(
    [
        log_reg_model_test_perf.T,  # Performance with the default threshold
        log_reg_model_test_perf_threshold.T,  # Performance with optimal threshold from ROC-AUC
        log_reg_model_test_perf_new_threshold.T,  # Performance with optimal threshold from Precision-Recall
    ],
    axis=1,
)

# Rename columns to reflect the threshold used
models_test_comp_df.columns = [
    f"Logistic Regression-default Threshold ({default_threshold_test})",
    f"Logistic Regression-Threshold from ROC-AUC ({optimal_threshold_auc_roc_test})",
    f"Logistic Regression-Threshold from Precision-Recall ({optimal_threshold_curve_test})",
]

# Display the comparison
print("Test performance comparison:")
display(models_test_comp_df)

Test performance comparison:

# Separate features and target variable
X = dfmodeling.drop(["booking_status"], axis=1)
Y = dfmodeling["booking_status"]

# Encode categorical variables using pd.get_dummies()
X = pd.get_dummies(X, drop_first=True)

# Splitting data into train and test sets (70:30 split with random_state=1)
X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size=0.3, random_state=1)

# Output the shapes and class distributions
print("Shape of Training set : ", X_train.shape)
print("Shape of test set : ", X_test.shape)

print("\nPercentage of classes in training set:")
display(y_train.value_counts(normalize=True))

print("\nPercentage of classes in test set:")
display(y_test.value_counts(normalize=True))

Shape of Training set :  (25392, 29)
Shape of test set :  (10883, 29)

Percentage of classes in training set:

booking_status
Not_Canceled   0.67064
Canceled       0.32936
Name: proportion, dtype: float64

Percentage of classes in test set:

booking_status
Not_Canceled   0.67638
Canceled       0.32362
Name: proportion, dtype: float64

# Building the decision tree model
model0 = DecisionTreeClassifier(random_state=1)
model0.fit(X_train, y_train)

DecisionTreeClassifier(random_state=1)

DecisionTreeClassifier(random_state=1)

# Convert y_train and y_test to numeric labels
y_train_numeric = y_train.map({"Canceled": 1, "Not_Canceled": 0})
y_test_numeric = y_test.map({"Canceled": 1, "Not_Canceled": 0})

# Function to compute different metrics to check performance of a classification model built using sklearn
def model_performance_classification_sklearn(model, predictors, target):
    """
    Function to compute different metrics to check classification model performance

    model: classifier
    predictors: independent variables
    target: dependent variable
    """

    # Predict using the independent variables
    pred = model.predict(predictors)

    # Convert predictions to numeric format if they are not already
    pred_numeric = pd.Series(pred).map({"Canceled": 1, "Not_Canceled": 0})

    # Calculate performance metrics
    acc = accuracy_score(target, pred_numeric)  # Compute Accuracy
    recall = recall_score(target, pred_numeric)  # Compute Recall
    precision = precision_score(target, pred_numeric)  # Compute Precision
    f1 = f1_score(target, pred_numeric)  # Compute F1-score

    # Creating a dataframe of metrics
    df_perf = pd.DataFrame(
        {
            "Accuracy": acc,
            "Recall": recall,
            "Precision": precision,
            "F1": f1,
        },
        index=[0],
    )

    return df_perf

# Function to plot the confusion_matrix of a classification model
def confusion_matrix_sklearn(model, predictors, target):
    """
    To plot the confusion_matrix with percentages.

    model: classifier
    predictors: independent variables
    target: dependent variable
    """
    # Predict using the model
    y_pred = model.predict(predictors)

    # Convert predictions to numeric format
    y_pred_numeric = pd.Series(y_pred).map({"Canceled": 1, "Not_Canceled": 0})

    # Generate the confusion matrix
    cm = confusion_matrix(target, y_pred_numeric)
    labels = np.asarray(
        [
            ["{0:0.0f}".format(item) + "\n{0:.2%}".format(item / cm.flatten().sum())]
            for item in cm.flatten()
        ]
    ).reshape(2, 2)

    # Plot the confusion matrix
    plt.figure(figsize=(6, 4))
    sns.heatmap(cm, annot=labels, fmt="")
    plt.ylabel("True label")
    plt.xlabel("Predicted label")
    plt.show()

# Create the confusion matrix for the training data using the numeric labels
confusion_matrix_sklearn(model0, X_train, y_train_numeric)

# Evaluate the model performance on the training data with numeric labels
decision_tree_perf_train = model_performance_classification_sklearn(
    model0, X_train, y_train_numeric
)

# Display the training performance metrics
print("Decision Tree Model Performance on Training Set:")
display(decision_tree_perf_train)

Decision Tree Model Performance on Training Set:

# Create the confusion matrix for the test data using the numeric labels
confusion_matrix_sklearn(model0, X_test, y_test_numeric)

# Evaluate the model performance on the test data with numeric labels
decision_tree_perf_test = model_performance_classification_sklearn(
    model0, X_test, y_test_numeric
)

# Display the test performance metrics
print("Decision Tree Model Performance on Test Set:")
display(decision_tree_perf_test)

Decision Tree Model Performance on Test Set:

# Get feature names from the training set
feature_names = list(X_train.columns)

# Get feature importances from the model
importances = model0.feature_importances_

# Sort the feature importances in ascending order
indices = np.argsort(importances)

# Plot the feature importances
plt.figure(figsize=(8, 8))
plt.title("Feature Importances")
plt.barh(range(len(indices)), importances[indices], color="violet", align="center")
plt.yticks(range(len(indices)), [feature_names[i] for i in indices])
plt.xlabel("Relative Importance")
plt.show()

# Choose the type of classifier.
estimator = DecisionTreeClassifier(random_state=1, class_weight="balanced")

# Grid of parameters to choose from
parameters = {
    "max_depth": np.arange(2, 7, 2),  # Depth of the tree
    "max_leaf_nodes": [50, 75, 150, 250],  # Maximum number of leaf nodes
    "min_samples_split": [
        10,
        30,
        50,
        70,
    ],  # Minimum number of samples required to split an internal node
}

# Type of scoring used to compare parameter combinations
acc_scorer = make_scorer(
    f1_score, pos_label="Canceled"
)  # F1 score focusing on the "Canceled" class

# Run the grid search
grid_obj = GridSearchCV(estimator, parameters, scoring=acc_scorer, cv=5)
grid_obj = grid_obj.fit(X_train, y_train)

# Set the clf to the best combination of parameters
best_estimator = grid_obj.best_estimator_

# Fit the best algorithm to the data
best_estimator.fit(X_train, y_train)

DecisionTreeClassifier(class_weight='balanced', max_depth=6, max_leaf_nodes=50,
                       min_samples_split=10, random_state=1)

DecisionTreeClassifier(class_weight='balanced', max_depth=6, max_leaf_nodes=50,
                       min_samples_split=10, random_state=1)

# Checking performance on the training set
confusion_matrix_sklearn(best_estimator, X_train, y_train_numeric)

# Evaluating the model performance on the training data
decision_tree_pruned_perf_train = model_performance_classification_sklearn(
    best_estimator, X_train, y_train_numeric
)

# Displaying the training performance metrics
print("Pruned Decision Tree Model Performance on Training Set:")
display(decision_tree_pruned_perf_train)

Pruned Decision Tree Model Performance on Training Set:

# Checking performance on the test set
confusion_matrix_sklearn(best_estimator, X_test, y_test_numeric)

# Evaluating the model performance on the test data
decision_tree_pruned_perf_test = model_performance_classification_sklearn(
    best_estimator, X_test, y_test_numeric
)

# Displaying the test performance metrics
print("Pruned Decision Tree Model Performance on Test Set:")
display(decision_tree_pruned_perf_test)

Pruned Decision Tree Model Performance on Test Set:

# Visualizing the pre-pruned decision tree
plt.figure(figsize=(20, 10))
out = tree.plot_tree(
    best_estimator,
    feature_names=X_train.columns,
    filled=True,
    fontsize=9,
    node_ids=False,
    class_names=None,
)

# Add arrows to the decision tree split if they are missing
for o in out:
    arrow = o.arrow_patch
    if arrow is not None:
        arrow.set_edgecolor("black")
        arrow.set_linewidth(1)

plt.show()

# Text report showing the rules of the decision tree
print(
    tree.export_text(
        best_estimator, feature_names=list(X_train.columns), show_weights=True
    )
)

|--- lead_time <= 151.50
|   |--- any_special_requests <= 0.50
|   |   |--- market_segment_type_Online <= 0.50
|   |   |   |--- lead_time <= 90.50
|   |   |   |   |--- total_nights <= 5.50
|   |   |   |   |   |--- avg_price_per_room <= 201.50
|   |   |   |   |   |   |--- weights: [419.00, 2739.90] class: Not_Canceled
|   |   |   |   |   |--- avg_price_per_room >  201.50
|   |   |   |   |   |   |--- weights: [25.81, 1.49] class: Canceled
|   |   |   |   |--- total_nights >  5.50
|   |   |   |   |   |--- avg_price_per_room <= 92.80
|   |   |   |   |   |   |--- weights: [12.14, 68.59] class: Not_Canceled
|   |   |   |   |   |--- avg_price_per_room >  92.80
|   |   |   |   |   |   |--- weights: [85.01, 17.15] class: Canceled
|   |   |   |--- lead_time >  90.50
|   |   |   |   |--- lead_time <= 117.50
|   |   |   |   |   |--- avg_price_per_room <= 93.58
|   |   |   |   |   |   |--- weights: [227.72, 214.72] class: Canceled
|   |   |   |   |   |--- avg_price_per_room >  93.58
|   |   |   |   |   |   |--- weights: [285.41, 82.76] class: Canceled
|   |   |   |   |--- lead_time >  117.50
|   |   |   |   |   |--- no_of_week_nights <= 1.50
|   |   |   |   |   |   |--- weights: [81.98, 87.23] class: Not_Canceled
|   |   |   |   |   |--- no_of_week_nights >  1.50
|   |   |   |   |   |   |--- weights: [48.58, 228.14] class: Not_Canceled
|   |   |--- market_segment_type_Online >  0.50
|   |   |   |--- lead_time <= 13.50
|   |   |   |   |--- avg_price_per_room <= 99.44
|   |   |   |   |   |--- arrival_month <= 1.50
|   |   |   |   |   |   |--- weights: [0.00, 92.45] class: Not_Canceled
|   |   |   |   |   |--- arrival_month >  1.50
|   |   |   |   |   |   |--- weights: [132.08, 363.83] class: Not_Canceled
|   |   |   |   |--- avg_price_per_room >  99.44
|   |   |   |   |   |--- lead_time <= 3.50
|   |   |   |   |   |   |--- weights: [85.01, 219.94] class: Not_Canceled
|   |   |   |   |   |--- lead_time >  3.50
|   |   |   |   |   |   |--- weights: [280.85, 132.71] class: Canceled
|   |   |   |--- lead_time >  13.50
|   |   |   |   |--- required_car_parking_space <= 0.50
|   |   |   |   |   |--- avg_price_per_room <= 71.92
|   |   |   |   |   |   |--- weights: [159.40, 158.80] class: Canceled
|   |   |   |   |   |--- avg_price_per_room >  71.92
|   |   |   |   |   |   |--- weights: [3543.28, 850.67] class: Canceled
|   |   |   |   |--- required_car_parking_space >  0.50
|   |   |   |   |   |--- weights: [1.52, 48.46] class: Not_Canceled
|   |--- any_special_requests >  0.50
|   |   |--- no_of_special_requests <= 1.50
|   |   |   |--- market_segment_type_Online <= 0.50
|   |   |   |   |--- lead_time <= 102.50
|   |   |   |   |   |--- type_of_meal_plan_Not Selected <= 0.50
|   |   |   |   |   |   |--- weights: [9.11, 697.09] class: Not_Canceled
|   |   |   |   |   |--- type_of_meal_plan_Not Selected >  0.50
|   |   |   |   |   |   |--- weights: [9.11, 15.66] class: Not_Canceled
|   |   |   |   |--- lead_time >  102.50
|   |   |   |   |   |--- no_of_week_nights <= 2.50
|   |   |   |   |   |   |--- weights: [19.74, 32.06] class: Not_Canceled
|   |   |   |   |   |--- no_of_week_nights >  2.50
|   |   |   |   |   |   |--- weights: [3.04, 44.73] class: Not_Canceled
|   |   |   |--- market_segment_type_Online >  0.50
|   |   |   |   |--- lead_time <= 8.50
|   |   |   |   |   |--- lead_time <= 4.50
|   |   |   |   |   |   |--- weights: [44.03, 498.03] class: Not_Canceled
|   |   |   |   |   |--- lead_time >  4.50
|   |   |   |   |   |   |--- weights: [63.76, 258.71] class: Not_Canceled
|   |   |   |   |--- lead_time >  8.50
|   |   |   |   |   |--- required_car_parking_space <= 0.50
|   |   |   |   |   |   |--- weights: [1451.32, 2512.51] class: Not_Canceled
|   |   |   |   |   |--- required_car_parking_space >  0.50
|   |   |   |   |   |   |--- weights: [1.52, 134.20] class: Not_Canceled
|   |   |--- no_of_special_requests >  1.50
|   |   |   |--- lead_time <= 90.50
|   |   |   |   |--- no_of_week_nights <= 3.50
|   |   |   |   |   |--- weights: [0.00, 1585.04] class: Not_Canceled
|   |   |   |   |--- no_of_week_nights >  3.50
|   |   |   |   |   |--- no_of_special_requests <= 2.50
|   |   |   |   |   |   |--- weights: [57.69, 180.42] class: Not_Canceled
|   |   |   |   |   |--- no_of_special_requests >  2.50
|   |   |   |   |   |   |--- weights: [0.00, 52.19] class: Not_Canceled
|   |   |   |--- lead_time >  90.50
|   |   |   |   |--- no_of_special_requests <= 2.50
|   |   |   |   |   |--- arrival_month <= 8.50
|   |   |   |   |   |   |--- weights: [56.17, 184.90] class: Not_Canceled
|   |   |   |   |   |--- arrival_month >  8.50
|   |   |   |   |   |   |--- weights: [106.27, 106.61] class: Not_Canceled
|   |   |   |   |--- no_of_special_requests >  2.50
|   |   |   |   |   |--- weights: [0.00, 67.10] class: Not_Canceled
|--- lead_time >  151.50
|   |--- avg_price_per_room <= 100.04
|   |   |--- any_special_requests <= 0.50
|   |   |   |--- no_of_adults <= 1.50
|   |   |   |   |--- market_segment_type_Online <= 0.50
|   |   |   |   |   |--- lead_time <= 163.50
|   |   |   |   |   |   |--- weights: [24.29, 3.73] class: Canceled
|   |   |   |   |   |--- lead_time >  163.50
|   |   |   |   |   |   |--- weights: [62.24, 257.96] class: Not_Canceled
|   |   |   |   |--- market_segment_type_Online >  0.50
|   |   |   |   |   |--- avg_price_per_room <= 2.50
|   |   |   |   |   |   |--- weights: [3.04, 8.95] class: Not_Canceled
|   |   |   |   |   |--- avg_price_per_room >  2.50
|   |   |   |   |   |   |--- weights: [97.16, 0.75] class: Canceled
|   |   |   |--- no_of_adults >  1.50
|   |   |   |   |--- avg_price_per_room <= 82.47
|   |   |   |   |   |--- market_segment_type_Offline <= 0.50
|   |   |   |   |   |   |--- weights: [282.37, 2.98] class: Canceled
|   |   |   |   |   |--- market_segment_type_Offline >  0.50
|   |   |   |   |   |   |--- weights: [385.60, 213.97] class: Canceled
|   |   |   |   |--- avg_price_per_room >  82.47
|   |   |   |   |   |--- no_of_adults <= 2.50
|   |   |   |   |   |   |--- weights: [1030.80, 23.86] class: Canceled
|   |   |   |   |   |--- no_of_adults >  2.50
|   |   |   |   |   |   |--- weights: [0.00, 5.22] class: Not_Canceled
|   |   |--- any_special_requests >  0.50
|   |   |   |--- no_of_weekend_nights <= 0.50
|   |   |   |   |--- lead_time <= 180.50
|   |   |   |   |   |--- lead_time <= 159.50
|   |   |   |   |   |   |--- weights: [7.59, 7.46] class: Canceled
|   |   |   |   |   |--- lead_time >  159.50
|   |   |   |   |   |   |--- weights: [4.55, 37.28] class: Not_Canceled
|   |   |   |   |--- lead_time >  180.50
|   |   |   |   |   |--- no_of_special_requests <= 2.50
|   |   |   |   |   |   |--- weights: [212.54, 20.13] class: Canceled
|   |   |   |   |   |--- no_of_special_requests >  2.50
|   |   |   |   |   |   |--- weights: [0.00, 8.95] class: Not_Canceled
|   |   |   |--- no_of_weekend_nights >  0.50
|   |   |   |   |--- market_segment_type_Offline <= 0.50
|   |   |   |   |   |--- arrival_month <= 11.50
|   |   |   |   |   |   |--- weights: [110.82, 231.12] class: Not_Canceled
|   |   |   |   |   |--- arrival_month >  11.50
|   |   |   |   |   |   |--- weights: [34.92, 19.38] class: Canceled
|   |   |   |   |--- market_segment_type_Offline >  0.50
|   |   |   |   |   |--- lead_time <= 348.50
|   |   |   |   |   |   |--- weights: [3.04, 106.61] class: Not_Canceled
|   |   |   |   |   |--- lead_time >  348.50
|   |   |   |   |   |   |--- weights: [4.55, 5.96] class: Not_Canceled
|   |--- avg_price_per_room >  100.04
|   |   |--- arrival_month <= 11.50
|   |   |   |--- no_of_special_requests <= 2.50
|   |   |   |   |--- weights: [3200.19, 0.00] class: Canceled
|   |   |   |--- no_of_special_requests >  2.50
|   |   |   |   |--- weights: [0.00, 23.11] class: Not_Canceled
|   |   |--- arrival_month >  11.50
|   |   |   |--- any_special_requests <= 0.50
|   |   |   |   |--- weights: [0.00, 35.04] class: Not_Canceled
|   |   |   |--- any_special_requests >  0.50
|   |   |   |   |--- arrival_date <= 24.50
|   |   |   |   |   |--- weights: [0.00, 3.73] class: Not_Canceled
|   |   |   |   |--- arrival_date >  24.50
|   |   |   |   |   |--- weights: [22.77, 3.73] class: Canceled

# Display the importances
importances = best_estimator.feature_importances_
print(importances)

[0.02702997 0.         0.01243347 0.00702817 0.01415876 0.4676694
 0.         0.01418297 0.00076349 0.         0.         0.
 0.08210707 0.03096242 0.00839254 0.13891175 0.         0.
 0.00095344 0.         0.         0.         0.         0.
 0.         0.         0.         0.01004782 0.18535873]

# Get the importance of each feature in the decision tree
importances = best_estimator.feature_importances_
indices = np.argsort(importances)

# Plotting the feature importances
plt.figure(figsize=(8, 8))
plt.title("Feature Importances")
plt.barh(range(len(indices)), importances[indices], color="violet", align="center")
plt.yticks(range(len(indices)), [feature_names[i] for i in indices])
plt.xlabel("Relative Importance")
plt.show()

# Initialize the classifier (ensure consistency with your previous settings)
clf = DecisionTreeClassifier(random_state=1, class_weight="balanced")

# Get the cost complexity pruning path
path = clf.cost_complexity_pruning_path(X_train, y_train_numeric)
ccp_alphas, impurities = abs(path.ccp_alphas), path.impurities

# Display the pruning path data
pruning_path_df = pd.DataFrame({"Alpha": ccp_alphas, "Total Impurity": impurities})
display(pruning_path_df)

# Plot the total impurity vs effective alpha
fig, ax = plt.subplots(figsize=(10, 5))
ax.plot(ccp_alphas[:-1], impurities[:-1], marker="o", drawstyle="steps-post")
ax.set_xlabel("Effective Alpha")
ax.set_ylabel("Total Impurity of Leaves")
ax.set_title("Total Impurity vs Effective Alpha for Training Set")
plt.show()

# Initialize list to store decision trees trained with different alpha values
clfs = []
for ccp_alpha in ccp_alphas:
    clf = DecisionTreeClassifier(
        random_state=1, ccp_alpha=ccp_alpha, class_weight="balanced"
    )
    clf.fit(X_train, y_train_numeric)
    clfs.append(clf)

# Print out the number of nodes in the last tree
print(
    "Number of nodes in the last tree is: {} with ccp_alpha: {}".format(
        clfs[-1].tree_.node_count, ccp_alphas[-1]
    )
)

Number of nodes in the last tree is: 1 with ccp_alpha: 0.08117914389137282

# Remove the trivial tree with only one node
clfs = clfs[:-1]
ccp_alphas = ccp_alphas[:-1]

# Calculate the number of nodes and the depth for each tree
node_counts = [clf.tree_.node_count for clf in clfs]
depth = [clf.tree_.max_depth for clf in clfs]

# Plot the results
fig, ax = plt.subplots(2, 1, figsize=(10, 7))

ax[0].plot(ccp_alphas, node_counts, marker="o", drawstyle="steps-post")
ax[0].set_xlabel("alpha")
ax[0].set_ylabel("number of nodes")
ax[0].set_title("Number of nodes vs alpha")

ax[1].plot(ccp_alphas, depth, marker="o", drawstyle="steps-post")
ax[1].set_xlabel("alpha")
ax[1].set_ylabel("depth of tree")
ax[1].set_title("Depth vs alpha")

fig.tight_layout()
plt.show()

# Calculate F1 Scores for training and testing sets
f1_train = []
f1_test = []

for clf in clfs:
    pred_train = clf.predict(X_train)
    f1_train.append(f1_score(y_train_numeric, pred_train))

    pred_test = clf.predict(X_test)
    f1_test.append(f1_score(y_test_numeric, pred_test))

# Plot F1 Score vs alpha for training and testing sets
fig, ax = plt.subplots(figsize=(15, 5))
ax.set_xlabel("alpha")
ax.set_ylabel("F1 Score")
ax.set_title("F1 Score vs alpha for training and testing sets")
ax.plot(ccp_alphas, f1_train, marker="o", label="train", drawstyle="steps-post")
ax.plot(ccp_alphas, f1_test, marker="o", label="test", drawstyle="steps-post")
ax.legend()
plt.show()

# Find the best alpha (the one that gives the highest F1 score on the test set)
index_best_model = np.argmax(f1_test)
best_model = clfs[index_best_model]
print("Best alpha:", ccp_alphas[index_best_model])
print("Best Decision Tree Model:", best_model)

Best alpha: 0.00010671446444450493
Best Decision Tree Model: DecisionTreeClassifier(ccp_alpha=0.00010671446444450493,
                       class_weight='balanced', random_state=1)

# Refit the best model without using the ccp_alpha parameter
best_model_unpruned = DecisionTreeClassifier(random_state=1, class_weight="balanced")
best_model_unpruned.fit(X_train, y_train)

# Check performance on the training set
confusion_matrix_sklearn(best_model_unpruned, X_train, y_train_numeric)
decision_tree_perf_train_unpruned = model_performance_classification_sklearn(
    best_model_unpruned, X_train, y_train_numeric
)
print("Decision Tree Model Performance on Training Set (Unpruned):")
display(decision_tree_perf_train_unpruned)

# Check performance on the test set
confusion_matrix_sklearn(best_model_unpruned, X_test, y_test_numeric)
decision_tree_perf_test_unpruned = model_performance_classification_sklearn(
    best_model_unpruned, X_test, y_test_numeric
)
print("Decision Tree Model Performance on Test Set (Unpruned):")
display(decision_tree_perf_test_unpruned)

Decision Tree Model Performance on Training Set (Unpruned):

Decision Tree Model Performance on Test Set (Unpruned):

# Set a fixed alpha value for pruning (choose a reasonable small value)
fixed_alpha = 0.001  # You can adjust this value based on your data

# Refit the decision tree with this fixed alpha
best_model_fixed_alpha = DecisionTreeClassifier(
    random_state=1, ccp_alpha=fixed_alpha, class_weight="balanced"
)
best_model_fixed_alpha.fit(X_train, y_train)

# Check performance on the training set
confusion_matrix_sklearn(best_model_fixed_alpha, X_train, y_train_numeric)
decision_tree_perf_train_fixed_alpha = model_performance_classification_sklearn(
    best_model_fixed_alpha, X_train, y_train_numeric
)
print("Decision Tree Model Performance on Training Set (Fixed Alpha):")
print(decision_tree_perf_train_fixed_alpha)

# Check performance on the test set
confusion_matrix_sklearn(best_model_fixed_alpha, X_test, y_test_numeric)
decision_tree_perf_test_fixed_alpha = model_performance_classification_sklearn(
    best_model_fixed_alpha, X_test, y_test_numeric
)
print("Decision Tree Model Performance on Test Set (Fixed Alpha):")
print(decision_tree_perf_test_fixed_alpha)

Decision Tree Model Performance on Training Set (Fixed Alpha):
   Accuracy  Recall  Precision      F1
0   0.85586 0.77998    0.78185 0.78092

Decision Tree Model Performance on Test Set (Fixed Alpha):
   Accuracy  Recall  Precision      F1
0   0.85271 0.77200    0.77266 0.77233

# Verify that there are no NaNs in the feature sets
print("NaNs in X_train:", X_train.isnull().sum().sum())
print("NaNs in X_test:", X_test.isnull().sum().sum())

# Verify the integrity of y_train and y_test
print("Unique values in y_train:", y_train.unique())
print("Unique values in y_test:", y_test.unique())

NaNs in X_train: 0
NaNs in X_test: 0
Unique values in y_train: ['Canceled' 'Not_Canceled']
Unique values in y_test: ['Not_Canceled' 'Canceled']

# Compare the performance of models
model_comparison_df = pd.concat(
    [
        decision_tree_perf_train_unpruned.T,  # Pre-pruned Decision Tree
        decision_tree_perf_train_fixed_alpha.T,  # Post-pruned Decision Tree
        log_reg_model_train_perf.T,  # Logistic Regression (from earlier)
    ],
    axis=1,
)
model_comparison_df.columns = [
    "Pre-pruned Decision Tree",
    "Post-pruned Decision Tree",
    "Logistic Regression",
]
print("Model Performance Comparison:")
display(model_comparison_df)

Model Performance Comparison:

plt.figure(figsize=(20, 10))
out = tree.plot_tree(
    best_model_fixed_alpha,
    feature_names=feature_names,
    filled=True,
    fontsize=9,
    node_ids=False,
    class_names=None,
)
for o in out:
    arrow = o.arrow_patch
    if arrow is not None:
        arrow.set_edgecolor("black")
        arrow.set_linewidth(1)
plt.show()

# Print the rules of the pruned decision tree
print(
    tree.export_text(
        best_model_fixed_alpha, feature_names=feature_names, show_weights=True
    )
)

|--- lead_time <= 151.50
|   |--- any_special_requests <= 0.50
|   |   |--- market_segment_type_Online <= 0.50
|   |   |   |--- lead_time <= 90.50
|   |   |   |   |--- total_nights <= 5.50
|   |   |   |   |   |--- avg_price_per_room <= 201.50
|   |   |   |   |   |   |--- no_of_weekend_nights <= 0.50
|   |   |   |   |   |   |   |--- market_segment_type_Offline <= 0.50
|   |   |   |   |   |   |   |   |--- weights: [132.08, 536.80] class: Not_Canceled
|   |   |   |   |   |   |   |--- market_segment_type_Offline >  0.50
|   |   |   |   |   |   |   |   |--- weights: [0.00, 1199.59] class: Not_Canceled
|   |   |   |   |   |   |--- no_of_weekend_nights >  0.50
|   |   |   |   |   |   |   |--- lead_time <= 74.50
|   |   |   |   |   |   |   |   |--- arrival_date <= 27.50
|   |   |   |   |   |   |   |   |   |--- weights: [141.18, 854.40] class: Not_Canceled
|   |   |   |   |   |   |   |   |--- arrival_date >  27.50
|   |   |   |   |   |   |   |   |   |--- lead_time <= 1.50
|   |   |   |   |   |   |   |   |   |   |--- weights: [53.13, 8.95] class: Canceled
|   |   |   |   |   |   |   |   |   |--- lead_time >  1.50
|   |   |   |   |   |   |   |   |   |   |--- weights: [4.55, 58.15] class: Not_Canceled
|   |   |   |   |   |   |   |--- lead_time >  74.50
|   |   |   |   |   |   |   |   |--- weights: [88.05, 82.01] class: Canceled
|   |   |   |   |   |--- avg_price_per_room >  201.50
|   |   |   |   |   |   |--- weights: [25.81, 1.49] class: Canceled
|   |   |   |   |--- total_nights >  5.50
|   |   |   |   |   |--- avg_price_per_room <= 92.80
|   |   |   |   |   |   |--- weights: [12.14, 68.59] class: Not_Canceled
|   |   |   |   |   |--- avg_price_per_room >  92.80
|   |   |   |   |   |   |--- weights: [85.01, 17.15] class: Canceled
|   |   |   |--- lead_time >  90.50
|   |   |   |   |--- lead_time <= 117.50
|   |   |   |   |   |--- avg_price_per_room <= 93.58
|   |   |   |   |   |   |--- avg_price_per_room <= 75.07
|   |   |   |   |   |   |   |--- no_of_week_nights <= 2.50
|   |   |   |   |   |   |   |   |--- weights: [154.85, 25.35] class: Canceled
|   |   |   |   |   |   |   |--- no_of_week_nights >  2.50
|   |   |   |   |   |   |   |   |--- weights: [15.18, 60.39] class: Not_Canceled
|   |   |   |   |   |   |--- avg_price_per_room >  75.07
|   |   |   |   |   |   |   |--- weights: [57.69, 128.98] class: Not_Canceled
|   |   |   |   |   |--- avg_price_per_room >  93.58
|   |   |   |   |   |   |--- weights: [285.41, 82.76] class: Canceled
|   |   |   |   |--- lead_time >  117.50
|   |   |   |   |   |--- weights: [130.56, 315.37] class: Not_Canceled
|   |   |--- market_segment_type_Online >  0.50
|   |   |   |--- lead_time <= 13.50
|   |   |   |   |--- avg_price_per_room <= 99.44
|   |   |   |   |   |--- weights: [132.08, 456.28] class: Not_Canceled
|   |   |   |   |--- avg_price_per_room >  99.44
|   |   |   |   |   |--- lead_time <= 3.50
|   |   |   |   |   |   |--- weights: [85.01, 219.94] class: Not_Canceled
|   |   |   |   |   |--- lead_time >  3.50
|   |   |   |   |   |   |--- arrival_month <= 8.50
|   |   |   |   |   |   |   |--- weights: [232.27, 61.14] class: Canceled
|   |   |   |   |   |   |--- arrival_month >  8.50
|   |   |   |   |   |   |   |--- weights: [48.58, 71.57] class: Not_Canceled
|   |   |   |--- lead_time >  13.50
|   |   |   |   |--- required_car_parking_space <= 0.50
|   |   |   |   |   |--- avg_price_per_room <= 71.92
|   |   |   |   |   |   |--- weights: [159.40, 158.80] class: Canceled
|   |   |   |   |   |--- avg_price_per_room >  71.92
|   |   |   |   |   |   |--- arrival_year <= 2017.50
|   |   |   |   |   |   |   |--- lead_time <= 65.50
|   |   |   |   |   |   |   |   |--- weights: [21.25, 87.23] class: Not_Canceled
|   |   |   |   |   |   |   |--- lead_time >  65.50
|   |   |   |   |   |   |   |   |--- weights: [110.82, 20.13] class: Canceled
|   |   |   |   |   |   |--- arrival_year >  2017.50
|   |   |   |   |   |   |   |--- weights: [3411.21, 743.32] class: Canceled
|   |   |   |   |--- required_car_parking_space >  0.50
|   |   |   |   |   |--- weights: [1.52, 48.46] class: Not_Canceled
|   |--- any_special_requests >  0.50
|   |   |--- no_of_special_requests <= 1.50
|   |   |   |--- market_segment_type_Online <= 0.50
|   |   |   |   |--- weights: [40.99, 789.54] class: Not_Canceled
|   |   |   |--- market_segment_type_Online >  0.50
|   |   |   |   |--- lead_time <= 8.50
|   |   |   |   |   |--- weights: [107.79, 756.73] class: Not_Canceled
|   |   |   |   |--- lead_time >  8.50
|   |   |   |   |   |--- required_car_parking_space <= 0.50
|   |   |   |   |   |   |--- avg_price_per_room <= 118.55
|   |   |   |   |   |   |   |--- lead_time <= 61.50
|   |   |   |   |   |   |   |   |--- weights: [337.02, 972.94] class: Not_Canceled
|   |   |   |   |   |   |   |--- lead_time >  61.50
|   |   |   |   |   |   |   |   |--- arrival_year <= 2017.50
|   |   |   |   |   |   |   |   |   |--- weights: [112.34, 46.97] class: Canceled
|   |   |   |   |   |   |   |   |--- arrival_year >  2017.50
|   |   |   |   |   |   |   |   |   |--- weights: [346.13, 680.69] class: Not_Canceled
|   |   |   |   |   |   |--- avg_price_per_room >  118.55
|   |   |   |   |   |   |   |--- weights: [655.83, 811.91] class: Not_Canceled
|   |   |   |   |   |--- required_car_parking_space >  0.50
|   |   |   |   |   |   |--- weights: [1.52, 134.20] class: Not_Canceled
|   |   |--- no_of_special_requests >  1.50
|   |   |   |--- lead_time <= 90.50
|   |   |   |   |--- weights: [57.69, 1817.66] class: Not_Canceled
|   |   |   |--- lead_time >  90.50
|   |   |   |   |--- weights: [162.44, 358.61] class: Not_Canceled
|--- lead_time >  151.50
|   |--- avg_price_per_room <= 100.04
|   |   |--- any_special_requests <= 0.50
|   |   |   |--- no_of_adults <= 1.50
|   |   |   |   |--- market_segment_type_Online <= 0.50
|   |   |   |   |   |--- weights: [86.53, 261.69] class: Not_Canceled
|   |   |   |   |--- market_segment_type_Online >  0.50
|   |   |   |   |   |--- weights: [100.20, 9.69] class: Canceled
|   |   |   |--- no_of_adults >  1.50
|   |   |   |   |--- avg_price_per_room <= 82.47
|   |   |   |   |   |--- market_segment_type_Offline <= 0.50
|   |   |   |   |   |   |--- weights: [282.37, 2.98] class: Canceled
|   |   |   |   |   |--- market_segment_type_Offline >  0.50
|   |   |   |   |   |   |--- arrival_month <= 11.50
|   |   |   |   |   |   |   |--- lead_time <= 244.00
|   |   |   |   |   |   |   |   |--- total_nights <= 2.50
|   |   |   |   |   |   |   |   |   |--- weights: [57.69, 4.47] class: Canceled
|   |   |   |   |   |   |   |   |--- total_nights >  2.50
|   |   |   |   |   |   |   |   |   |--- weights: [27.33, 104.38] class: Not_Canceled
|   |   |   |   |   |   |   |--- lead_time >  244.00
|   |   |   |   |   |   |   |   |--- arrival_year <= 2017.50
|   |   |   |   |   |   |   |   |   |--- weights: [0.00, 25.35] class: Not_Canceled
|   |   |   |   |   |   |   |   |--- arrival_year >  2017.50
|   |   |   |   |   |   |   |   |   |--- weights: [300.59, 33.55] class: Canceled
|   |   |   |   |   |   |--- arrival_month >  11.50
|   |   |   |   |   |   |   |--- weights: [0.00, 46.22] class: Not_Canceled
|   |   |   |   |--- avg_price_per_room >  82.47
|   |   |   |   |   |--- weights: [1030.80, 29.08] class: Canceled
|   |   |--- any_special_requests >  0.50
|   |   |   |--- no_of_weekend_nights <= 0.50
|   |   |   |   |--- lead_time <= 180.50
|   |   |   |   |   |--- weights: [12.14, 44.73] class: Not_Canceled
|   |   |   |   |--- lead_time >  180.50
|   |   |   |   |   |--- weights: [212.54, 29.08] class: Canceled
|   |   |   |--- no_of_weekend_nights >  0.50
|   |   |   |   |--- weights: [153.33, 363.08] class: Not_Canceled
|   |--- avg_price_per_room >  100.04
|   |   |--- arrival_month <= 11.50
|   |   |   |--- no_of_special_requests <= 2.50
|   |   |   |   |--- weights: [3200.19, 0.00] class: Canceled
|   |   |   |--- no_of_special_requests >  2.50
|   |   |   |   |--- weights: [0.00, 23.11] class: Not_Canceled
|   |   |--- arrival_month >  11.50
|   |   |   |--- weights: [22.77, 42.50] class: Not_Canceled

importances = best_model_fixed_alpha.feature_importances_
indices = np.argsort(importances)

plt.figure(figsize=(8, 8))
plt.title("Feature Importances (Pruned Tree)")
plt.barh(range(len(indices)), importances[indices], color="violet", align="center")
plt.yticks(range(len(indices)), [feature_names[i] for i in indices])
plt.xlabel("Relative Importance")
plt.show()

# Making a list of all numerical variables ('int64', 'float64', 'complex')
num_cols = df_eda.select_dtypes(include=["int64", "float64", "complex"]).columns

# Iterate through each numerical column and plot the histogram and boxplot combined
for column in num_cols:
    print(f"Distribution of '{column}'")
    print(df_eda[column].describe())
    histogram_boxplot(df_eda, column, bins=50, kde=True)
    print("-" * 100)

Distribution of 'no_of_adults'
count   36275.00000
mean        1.84496
std         0.51871
min         0.00000
25%         2.00000
50%         2.00000
75%         2.00000
max         4.00000
Name: no_of_adults, dtype: float64

----------------------------------------------------------------------------------------------------
Distribution of 'no_of_children'
count   36275.00000
mean        0.10528
std         0.40265
min         0.00000
25%         0.00000
50%         0.00000
75%         0.00000
max        10.00000
Name: no_of_children, dtype: float64

----------------------------------------------------------------------------------------------------
Distribution of 'no_of_weekend_nights'
count   36275.00000
mean        0.81072
std         0.87064
min         0.00000
25%         0.00000
50%         1.00000
75%         2.00000
max         7.00000
Name: no_of_weekend_nights, dtype: float64

----------------------------------------------------------------------------------------------------
Distribution of 'no_of_week_nights'
count   36275.00000
mean        2.20430
std         1.41090
min         0.00000
25%         1.00000
50%         2.00000
75%         3.00000
max        17.00000
Name: no_of_week_nights, dtype: float64

----------------------------------------------------------------------------------------------------
Distribution of 'required_car_parking_space'
count   36275.00000
mean        0.03099
std         0.17328
min         0.00000
25%         0.00000
50%         0.00000
75%         0.00000
max         1.00000
Name: required_car_parking_space, dtype: float64

----------------------------------------------------------------------------------------------------
Distribution of 'lead_time'
count   36275.00000
mean       85.23256
std        85.93082
min         0.00000
25%        17.00000
50%        57.00000
75%       126.00000
max       443.00000
Name: lead_time, dtype: float64

----------------------------------------------------------------------------------------------------
Distribution of 'arrival_year'
count   36275.00000
mean     2017.82043
std         0.38384
min      2017.00000
25%      2018.00000
50%      2018.00000
75%      2018.00000
max      2018.00000
Name: arrival_year, dtype: float64

----------------------------------------------------------------------------------------------------
Distribution of 'arrival_month'
count   36275.00000
mean        7.42365
std         3.06989
min         1.00000
25%         5.00000
50%         8.00000
75%        10.00000
max        12.00000
Name: arrival_month, dtype: float64

----------------------------------------------------------------------------------------------------
Distribution of 'arrival_date'
count   36275.00000
mean       15.59700
std         8.74045
min         1.00000
25%         8.00000
50%        16.00000
75%        23.00000
max        31.00000
Name: arrival_date, dtype: float64

----------------------------------------------------------------------------------------------------
Distribution of 'repeated_guest'
count   36275.00000
mean        0.02564
std         0.15805
min         0.00000
25%         0.00000
50%         0.00000
75%         0.00000
max         1.00000
Name: repeated_guest, dtype: float64

# Making a list of all categorical variables ('object' or 'category')
cat_cols = df_eda.select_dtypes(include=["object", "category"]).columns

# Iterate through each categorical column and plot the distribution
for column in cat_cols:
    print("Distribution of '", column, "'")
    print(df_eda[column].describe())
    labeled_barplot(df_eda, column, perc=True)
    print("-" * 100)

Distribution of ' type_of_meal_plan '
count           36275
unique              4
top       Meal Plan 1
freq            27835
Name: type_of_meal_plan, dtype: object

----------------------------------------------------------------------------------------------------
Distribution of ' room_type_reserved '
count           36275
unique              7
top       Room_Type 1
freq            28130
Name: room_type_reserved, dtype: object

----------------------------------------------------------------------------------------------------
Distribution of ' market_segment_type '
count      36275
unique         5
top       Online
freq       23214
Name: market_segment_type, dtype: object

----------------------------------------------------------------------------------------------------
Distribution of ' booking_status '
count            36275
unique               2
top       Not_Canceled
freq             24390
Name: booking_status, dtype: object

----------------------------------------------------------------------------------------------------

cols_list = df_eda.select_dtypes(include=np.number).columns.tolist()

plt.figure(figsize=(15, 7))
sns.heatmap(
    df[cols_list].corr(), annot=True, vmin=-1, vmax=1, fmt=".2f", cmap="Spectral"
)
plt.show()

	count	mean	std	min	25%	50%	75%	max
market_segment_type
Aviation	125.00000	100.70400	8.53836	79.00000	95.00000	95.00000	110.00000	110.00000
Complementary	391.00000	3.14176	15.51297	0.00000	0.00000	0.00000	0.00000	170.00000
Corporate	2017.00000	82.91174	23.69000	31.00000	65.00000	79.00000	95.00000	220.00000
Offline	10528.00000	91.63268	24.99560	12.00000	75.00000	90.00000	109.00000	540.00000
Online	23214.00000	112.25685	35.22032	0.00000	89.00000	107.10000	131.75000	375.50000

	no_of_children	no_of_weekend_nights	no_of_week_nights	required_car_parking_space	lead_time	arrival_month	arrival_date	repeated_guest	no_of_previous_cancellations	no_of_special_requests	type_of_meal_plan_encoded	room_type_reserved_encoded
Odds	1.36309	1.04680	0.94883	0.29167	1.01159	0.87684	0.97798	0.03397	1.22039	0.37133	1.16121	1.18269
Change_odd%	36.30887	4.67955	-5.11741	-70.83330	1.15924	-12.31648	-2.20164	-96.60299	22.03925	-62.86743	16.12083	18.26885

	Logistic Regression-default Threshold (0.5)	Logistic Regression-Threshold from ROC-AUC (0.36428382207660154)	Logistic Regression-Threshold from Precision-Recall (0.3463375314218031)
Accuracy	0.75589	0.73529	0.72691
Recall	0.52772	0.71634	0.73968
Precision	0.65672	0.57586	0.56194
F1	0.58520	0.63846	0.63867

	Logistic Regression-default Threshold (0.5)	Logistic Regression-Threshold from ROC-AUC (0.3463375314218031)	Logistic Regression-Threshold from Precision-Recall (0.3576726321099713)
Accuracy	0.75369	0.73067	0.73880
Recall	0.52235	0.74172	0.72765
Precision	0.66596	0.57399	0.58698
F1	0.58548	0.64717	0.64979

	Alpha	Total Impurity
0	0.00000	0.00838
1	0.00000	0.00838
2	0.00000	0.00838
3	0.00000	0.00838
4	0.00000	0.00838
...	...	...
1841	0.00890	0.32806
1842	0.00980	0.33786
1843	0.01272	0.35058
1844	0.03412	0.41882
1845	0.08118	0.50000

	Booking_ID	no_of_adults	no_of_weekend_nights	no_of_week_nights	type_of_meal_plan	room_type_reserved	lead_time	arrival_year	arrival_month	arrival_date	market_segment_type	avg_price_per_room	no_of_special_requests	booking_status
0	INN00001	2	1	2	Meal Plan 1	Room_Type 1	224	2017	10	2	Offline	65.00000	0	Not_Canceled
1	INN00002	2	2	3	Not Selected	Room_Type 1	5	2018	11	6	Online	106.68000	1	Not_Canceled
2	INN00003	1	2	1	Meal Plan 1	Room_Type 1	1	2018	2	28	Online	60.00000	0	Canceled
3	INN00004	2	0	2	Meal Plan 1	Room_Type 1	211	2018	5	20	Online	100.00000	0	Canceled
4	INN00005	2	1	1	Not Selected	Room_Type 1	48	2018	4	11	Online	94.50000	0	Canceled

	Booking_ID	no_of_adults	no_of_weekend_nights	no_of_week_nights	type_of_meal_plan	room_type_reserved	lead_time	arrival_year	arrival_month	arrival_date	market_segment_type	avg_price_per_room	no_of_special_requests	booking_status
36270	INN36271	3	2	6	Meal Plan 1	Room_Type 4	85	2018	8	3	Online	167.80000	1	Not_Canceled
36271	INN36272	2	1	3	Meal Plan 1	Room_Type 1	228	2018	10	17	Online	90.95000	2	Canceled
36272	INN36273	2	2	6	Meal Plan 1	Room_Type 1	148	2018	7	1	Online	98.39000	2	Not_Canceled
36273	INN36274	2	0	3	Not Selected	Room_Type 1	63	2018	4	21	Online	94.50000	0	Canceled
36274	INN36275	2	1	2	Meal Plan 1	Room_Type 1	207	2018	12	30	Offline	161.67000	0	Not_Canceled

	count	mean	std	min	25%	50%	75%	max
no_of_adults	36275.00000	1.84496	0.51871	0.00000	2.00000	2.00000	2.00000	4.00000
no_of_children	36275.00000	0.10528	0.40265	0.00000	0.00000	0.00000	0.00000	10.00000
no_of_weekend_nights	36275.00000	0.81072	0.87064	0.00000	0.00000	1.00000	2.00000	7.00000
no_of_week_nights	36275.00000	2.20430	1.41090	0.00000	1.00000	2.00000	3.00000	17.00000
required_car_parking_space	36275.00000	0.03099	0.17328	0.00000	0.00000	0.00000	0.00000	1.00000
lead_time	36275.00000	85.23256	85.93082	0.00000	17.00000	57.00000	126.00000	443.00000
arrival_year	36275.00000	2017.82043	0.38384	2017.00000	2018.00000	2018.00000	2018.00000	2018.00000
arrival_month	36275.00000	7.42365	3.06989	1.00000	5.00000	8.00000	10.00000	12.00000
arrival_date	36275.00000	15.59700	8.74045	1.00000	8.00000	16.00000	23.00000	31.00000
repeated_guest	36275.00000	0.02564	0.15805	0.00000	0.00000	0.00000	0.00000	1.00000
no_of_previous_cancellations	36275.00000	0.02335	0.36833	0.00000	0.00000	0.00000	0.00000	13.00000
no_of_previous_bookings_not_canceled	36275.00000	0.15341	1.75417	0.00000	0.00000	0.00000	0.00000	58.00000
avg_price_per_room	36275.00000	103.42354	35.08942	0.00000	80.30000	99.45000	120.00000	540.00000
no_of_special_requests	36275.00000	0.61966	0.78624	0.00000	0.00000	0.00000	1.00000	5.00000

	count	unique	top	freq
type_of_meal_plan	36275	4	Meal Plan 1	27835
room_type_reserved	36275	7	Room_Type 1	28130
market_segment_type	36275	5	Online	23214
booking_status	36275	2	Not_Canceled	24390

	count	mean	std	min	25%	50%	75%	max
booking_status
Canceled	11885.00000	0.33462	0.57435	0.00000	0.00000	0.00000	1.00000	2.00000
Not_Canceled	24390.00000	0.75855	0.83653	0.00000	0.00000	1.00000	1.00000	5.00000

	no_of_adults	no_of_children	no_of_weekend_nights	no_of_week_nights	lead_time	arrival_year	arrival_month	arrival_date	avg_price_per_room	no_of_special_requests	any_special_requests	market_segment_type_encoded
25629	2	1	2	1	26	2017	10	17	161.00000	0	0	4
14473	2	1	1	1	98	2018	7	16	121.50000	2	1	4
23720	2	0	0	3	433	2018	9	8	70.00000	0	0	3
5843	2	0	2	5	195	2018	8	8	72.25000	0	0	3
18709	1	0	0	2	188	2018	6	15	130.00000	0	0	3

	Pre-pruned Decision Tree	Post-pruned Decision Tree	Logistic Regression
Accuracy	0.99311	0.85586	0.75589
Recall	0.99510	0.77998	0.52772
Precision	0.98415	0.78185	0.65672
F1	0.98960	0.78092	0.58520

Data Science and Business Analytics

Project 4 - Supervised Learning Classification: INN Hotels

Jorge Ramon Vazquez Campero

1. Problem Statement and Business Context¶

1.1 Business Context¶

1.2 Objective¶

1.3 Data Description¶

Importing necessary libraries and data¶

2. Data Overview¶

3. Exploratory Data Analysis (EDA)¶

1. What are the busiest months in the hotel?¶

2. Which market segment do most of the guests come from?¶

3. Hotel rates are dynamic and change according to demand and customer demographics. What are the differences in room prices in different market segments?¶

4. What percentage of bookings are canceled?¶

5. Repeating guests are the guests who stay in the hotel often and are important to brand equity. What percentage of repeating guests cancel?¶

6. Many guests have special requirements when booking a hotel room. Do these requirements affect booking cancellation?¶

4. Data Preprocessing¶

4.1 Missing Value & Duplicates Check and Treatment¶

4.2 Feature Engineering¶

4.3 Outlier Check and Treatment¶

4.4 Preparing Data for Modeling¶

4.5 Checking Multicollinearity (Optional if Outlier treatments were performed)¶

5. Logistic Regression Model Building¶

5.1 Model Evaluation Criteria¶

Which Case is More Important?¶

How to Reduce Losses?¶

To streamline our workflow and avoid repeating code for each model, we will create two essential functions that will handle the calculation of different metrics and the generation of a confusion matrix.¶

5.2 Logistic Regression Model¶

Checking Multicollinearity¶

Treating High P-values¶

Coefficient interpretations¶

Converting coefficients to odds¶

Model performance evaluation¶

Checking model performance on training set¶

ROC-AUC on Training Set¶

Model Performance Improvement¶

Let's use Precision-Recall curve and see if we can find a better threshold¶

Checking model performance on test set¶

ROC-AUC on Test Set¶

Option 1: Adjust Threshold Using the Optimal Threshold Found in the Training Set (0.3643)¶

Option 2: Determine a New Optimal Threshold for the Test Set¶

Final Logistic Model Performance¶

Training performance comparison¶

Test performance comparison¶

6. Decision Tree Model Building¶

6.1 Data Preparation for modeling (Decision Tree)¶

6.2 Decision Tree (default)¶

6.3 Model Evaluation Criterion¶

Which Case is More Important?¶

How to Reduce the Losses?¶

6.4 Streamlining Model Evaluation with Functions¶

Checking model performance on training set¶

Checking model performance on test set¶

Do we need to prune the tree?¶

6.5 Pruning the Tree¶

Pre-pruning the Tree¶

Checking performance on training set¶

Checking performance on test set¶

6.6 Model Visualization¶

6.7 Cost Complexity Pruning¶

6.8 F1 Score vs alpha for training and testing sets¶

6.9 Checking performance on Training and Test set¶

Model Performance Comparison and Conclusions¶

Actionable Insights and Recommendations¶

1. Lead Time is the Most Significant Factor¶

2. Special Requests as a Predictor¶

3. Market Segment Type¶

4. Price Sensitivity¶

5. Profitable Cancellation and Refund Policies¶

6. Other Recommendations¶

Appendix: Detailed Exploratory Data Analysis (EDA)

3.1 Univariate Analysis¶

3.1.1 Analysis of Numerical Features¶

3.1.2 Analysis of Categorical Features¶

Multivariate analysis¶